{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "e5ksBr8d6uhy"
   },
   "source": [
    "# 3.12 权重衰减\n",
    "\n",
    "## 3.12.1 方法\n",
    "\n",
    "## 3.12.2 高维线性回归实验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:43:47.024604Z",
     "start_time": "2020-04-12T05:43:47.019618Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers, models, initializers, optimizers, regularizers\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:42:52.691681Z",
     "start_time": "2020-04-12T05:42:52.409437Z"
    }
   },
   "outputs": [],
   "source": [
    "n_train, n_test, num_inputs = 20, 100, 200\n",
    "true_w, true_b = tf.ones((num_inputs, 1)) * 0.01, 0.05\n",
    "\n",
    "features = tf.random.normal(shape=(n_train + n_test, num_inputs))\n",
    "labels = tf.keras.backend.dot(features, true_w) + true_b\n",
    "labels += tf.random.normal(mean=0.01, shape=labels.shape)\n",
    "train_features, test_features = features[:n_train, :], features[n_train:, :]\n",
    "train_labels, test_labels = labels[:n_train], labels[n_train:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lmtuMj-XZOMG"
   },
   "source": [
    "## 3.12.3 从零开始实现\n",
    "\n",
    "### 3.12.3.1 初始化模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:42:53.757829Z",
     "start_time": "2020-04-12T05:42:53.753840Z"
    }
   },
   "outputs": [],
   "source": [
    "def init_params():\n",
    "    w = tf.Variable(tf.random.normal(mean=1, shape=(num_inputs, 1)))\n",
    "    b = tf.Variable(tf.zeros(shape=(1,)))\n",
    "    return [w, b]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gnOCGqteZ-no"
   },
   "source": [
    "### 3.12.3.2 定义L2范数惩罚项"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:42:54.666399Z",
     "start_time": "2020-04-12T05:42:54.662409Z"
    }
   },
   "outputs": [],
   "source": [
    "def l2_penalty(w):\n",
    "    return tf.reduce_sum((w**2)) / 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r5MT50ODbNhh"
   },
   "source": [
    "### 3.12.3.3 定义训练和测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:42:56.153461Z",
     "start_time": "2020-04-12T05:42:56.146438Z"
    }
   },
   "outputs": [],
   "source": [
    "def linreg(X, w, b):\n",
    "    return tf.matmul(X, w) + b\n",
    "\n",
    "def sgd(params, lr, batch_size, grads):\n",
    "    \"\"\"Mini-batch stochastic gradient descent.\"\"\"\n",
    "    for i, param in enumerate(params):\n",
    "        param.assign_sub(lr * grads[i] / batch_size)\n",
    "\n",
    "def set_figsize(figsize=(3.5, 2.5)):\n",
    "    \"\"\"Set matplotlib figure size.\"\"\"\n",
    "    plt.rcParams['figure.figsize'] = figsize\n",
    "\n",
    "def semilogy(x_vals, y_vals, x_label, y_label, x2_vals=None, y2_vals=None,\n",
    "             legend=None, figsize=(3.5, 2.5)):\n",
    "    \"\"\"Plot x and log(y).\"\"\"\n",
    "    set_figsize(figsize)\n",
    "    plt.xlabel(x_label)\n",
    "    plt.ylabel(y_label)\n",
    "    plt.semilogy(x_vals, y_vals)\n",
    "    if x2_vals and y2_vals:\n",
    "        plt.semilogy(x2_vals, y2_vals, linestyle=':')\n",
    "        plt.legend(legend)\n",
    "    plt.show()\n",
    "    \n",
    "def squared_loss(y_hat, y): \n",
    "    # 注意这里返回的是向量, 另外, pytorch里的MSELoss并没有除以 2\n",
    "    return (y_hat - tf.reshape(y, y_hat.shape)) ** 2 / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:42:57.286818Z",
     "start_time": "2020-04-12T05:42:57.275847Z"
    }
   },
   "outputs": [],
   "source": [
    "batch_size, num_epochs, lr = 1, 100, 0.003\n",
    "net, loss = linreg, squared_loss\n",
    "optimizer = tf.keras.optimizers.SGD()\n",
    "train_iter = tf.data.Dataset.from_tensor_slices(\n",
    "    (train_features, train_labels)).batch(batch_size).shuffle(batch_size)\n",
    "\n",
    "def fit_and_plot(lambd):\n",
    "    w, b = init_params()\n",
    "    train_ls, test_ls = [], []\n",
    "    for _ in range(num_epochs):\n",
    "        for X, y in train_iter:\n",
    "            with tf.GradientTape(persistent=True) as tape:\n",
    "                # 添加了L2范数惩罚项\n",
    "                l = loss(net(X, w, b), y) + lambd * l2_penalty(w)\n",
    "            grads = tape.gradient(l, [w, b])\n",
    "            sgd([w, b], lr, batch_size, grads)\n",
    "        train_ls.append(tf.reduce_mean(loss(net(train_features, w, b),\n",
    "                             train_labels)).numpy())\n",
    "        test_ls.append(tf.reduce_mean(loss(net(test_features, w, b),\n",
    "                            test_labels)).numpy())\n",
    "    semilogy(range(1, num_epochs + 1), train_ls, 'epochs', 'loss',\n",
    "                 range(1, num_epochs + 1), test_ls, ['train', 'test'])\n",
    "    print('L2 norm of w:', tf.norm(w).numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sj0gZgkltES_"
   },
   "source": [
    "### 3.12.3.4 观察过拟合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:43:03.815866Z",
     "start_time": "2020-04-12T05:42:59.180751Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAC1CAYAAAC56YFuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAeZElEQVR4nO3deXhV9bXw8e86mSdCCAECARKCIjJDUAZRaAvi0FvnAW2rdW57O9zbXuXWDr7P7VvvvX37WmvFEW2v1qHaqkWKFEUGJyAMEhAMMyGQkJCQeV73j70zgElIQs7ZJ8n6PM95OGufvc9eZ+ecxW9Pv5+oKsYYA+DzOgFjTPCwgmCMaWIFwRjTxAqCMaaJFQRjTBMrCMaYJqFeJxAIAwcO1NTUVK/TMCYoZGZmFqhqUmuv9YmCkJqayqZNm7xOw5igICIH23rNdhmMMU2sIBhjmvTIgiAiV4nI0yLypogs8DofY3qLoCkIIrJURPJFJOu06QtFZLeI7BGRBwBU9Q1VvQu4DbixWxJY9RCs+a/m+B8/h3W/aY7f+Ql8+LvmeMVi+PiJ5nj5j2HjM83xsh/Cpuea4ze/C1teaI7/eh9se7k5fv0u2P5ac/zn22HHG87z+jon/uxvTlxb6cS7VzhxdakTZ69y4soiJ9672onLC5z4wHonLj3mxAc/cuLiw06c4x5nObHfiXO3OHHBHic+tt2J83c5cf4uJz6W5cQF2U6cu8WJT+x34pxNTlx82IkPfezEJUed+MB6Jy4vcOJ97ztxxQknzl7lxNWlTrx7hRPXVjrxZ8ucuL7WiXe84cSNtr8Gr9/ZHG972dn+jba84Px9GmU+D3/7QXO88Rnn79vokyfh7w80xx8+BisfbI7XPwKrftEcr/01vPcfzfH7/+k8Gr33S1j7383xql8479Fo5YPOOhqd6bt3FoLpoOLzwGPAHxsniEgI8HtgPpADbBSRt1R1pzvLg+7rZ+/kYQiLbo5P7IOYFgdiC/c2f+EAju8GaVFPj++C8JjmOP8ziB54atx/RHOclwVJ554aJ09qjo9thxEznefa4MRpc06N07/kxA11TlxR6MT1blzp/qDqa9y4yInrqpy46qQbV7txsRPXVjpx4w+wttyNy5y4xo1ryp24utSJaytOi90fbFWxE9dVO3GlG9c3xkVuXOPEFYVO3FB/WlznxOXHnVgb3Djfjd0b9crym4sXOAUwb0eL+KizvRuV5Dp/n0Ynjzh/z6Y459S4+BAUfH5qXNziOF3RAagoaBHvh5qK5vjEPk5RuAfCo0+NW353CrKhrqY5PtN37yxIMN3tKCKpwDJVHe/GM4FfqOqlbrzYnfVh9/EPVV11pvfNyMhQO8tgjENEMlU1o7XXgmaXoQ3DgMMt4hx32j8DXwGuE5F7W1tQRO4WkU0isun48eP+z9SYXiCYdhlaI61MU1V9FHi0vQVV9SngKXBaCH7IzZheJ9hbCDnA8BZxCpDrUS7G9HrBXhA2AueISJqIhAM3AW95nJMxvVbQFAQReQn4CBgjIjkicoeq1gHfBd4BPgNeVdUd7b1PV/zxowNcu+RDgukAqzFeCJpjCKp6cxvTlwPL/bnu8BAfmQeL2JFbwvhh8f5clTFBLWhaCF5aMG4IIT5h+fajXqdijKesIAADYsKZlZ7I29uP2m6D6dOsILiumJDMwcIKduSWeJ2KMZ6xguCy3QZjrCA0GRATzuzRA3lzay71DbbbYPomKwgt3JgxnCPFlazNtkudTd9kBaGF+ecPZmBsOC99csjrVIzxhBWEFsJDfVyfMZx3d+WTV1LldTrGBJwVhNPcNH049Q3KyxsOn3lmY3oZKwinGZkYwyXnJvGnDQeprW/wOh1jAsoKQiu+OWskeSXVvLPjmNepGBNQVhBaMffcQYxMjOYPHx7wOhVjAsoKQit8PuHrM0ay8UARWUdOep2OMQFjBaEN12cMJyosxFoJpk+xgtCG+Kgwrp02jDe35VJYVu11OsYEhBWEdtw2K42augb+ZBcqmT7CCkI7Rg+K5eJzk/ifjw9SU2enIE3v1yMLgoiMEpFnReS1M899dm6fnUp+aTV/z7K7IE3vF/CC0Jkh29qiqvtU9Q7/Zuq45JwkRiXF8Oz6/dZ5iun1vGghPA8sbDmhxZBtlwHnAzeLyPkiMkFElp32GBTIZH0+4fbZaXyac5LNh4oCuWpjAi7gBUFV1wInTpt8AbDH/Z+/BngZ+JqqblfVK0975Ac652unDiM+Koxn1+8P9KqNCahgOYbQ1pBtrRKRRBF5ApjSYrzH0+fptqHcosNDufmCEazIOsbhExVnXsCYHipYCkKrQ7a1NbOqFqrqvaqarqq/amOep1Q1Q1UzkpKSWpulU74xcyQiYhcqmV4tWApC0A/ZNrR/FFdMSOaVjYcpq67zOh1j/CJYCkKPGLLtjovSKK2u49WN1leC6Z28OO3o2ZBtZ2vS8P5kjExg6Qf7rSNW0yt5cZbhZlVNVtUwVU1R1Wfd6ctV9Vz3uMAvA51XR905J42cokpWWl8JphcKll2GHmP++UMYMSCap9ft8zoVY7qdFYROCvEJ35qdyuZDxWQetAuVTO9iBaELrs8YTr/IUJ6xVoLpZawgdEFMRCiLLhzJOzuOcajQLlQyvYcVhC66bVYqIT5h6Qd2ObPpPawgdNGQ+Ei+Omkor2w8THFFjdfpGNMtrCCchbvmjKKytp4XrUcl00tYQTgLY5P7MeecgTz3wQGq6+q9TseYs2YF4Szdc3E6BWXVvLklqG69MKZLrCCcpdmjEzk/uR9PrdtHg13ObHq4UK8T6OlEhLsvHsUPXtnK6t35fHnsYK9TMu2ora0lJyeHqqreP7p3ZGQkKSkphIWFdXgZKwjd4IqJyfzXil08uXafFYQgl5OTQ1xcHKmpqYi01g1H76CqFBYWkpOTQ1paWoeXs12GbhAW4uOOOaPYsP8EW6zfxaBWVVVFYmJiry4G4LRcExMTO90SsoLQTW6aPpz4qDCeXGOXMwe73l4MGnXlc1pB6CYxEaF8fcZI3tl5jH3Hy7xOx5gusYLQjb45K5WwEB9Pr7PLmU3riouLefzxxzu93OWXX05xcbEfMjpVjy0IIhIjIpkicqXXuTRKiovg+mkpvL45h/zS3n8U23ReWwWhvr79C9uWL19O//79/ZVWkx45cpPrfuBV/2TZdXfNGUVtfQPPfXDA61RMEHrggQfYu3cvkydPZvr06cybN49FixYxYcIEAK666iqmTZvGuHHjeOqpp5qWS01NpaCggAMHDjB27Fjuuusuxo0bx4IFC6isrOy2/Lw47fg88Bjwx8YJLUZumo/TA/NGEXkLCAFO72b9W8BEYCcQGYB8OyV1YAyXj0/mhY8P8u256cRFdvwcsAmsh/62g525Jd36nucP7cfPvzquzdcffvhhsrKy2Lp1K++//z5XXHEFWVlZTacGly5dyoABA6isrGT69Olce+21JCYmnvIe2dnZvPTSSzz99NPccMMNvP7669x6663dkn9PHblpHjADWATcJSJBtetz7yXplFbV2TDy5owuuOCCU64TePTRR5k0aRIzZszg8OHDZGdnf2GZtLQ0Jk+eDMC0adM4cOBAt+XToRaCiHwfeA4oBZ4BpgAPqOrKbsqjtZGbLmxrZlX9iZvXbUCBqn5hrHYRuRu4G2DEiBHdlGbHTEiJZ/boRJ5dv5/bZqcSERoS0PWbjmnvf/JAiYmJaXr+/vvvs2rVKj766COio6OZO3duq9cRREREND0PCQnp1l2Gjv7P+i1VLQEWAEnA7cDD3ZZFJ0duappB9XlVXdbGa906clNn3XfJaPJLq/nL5iMBX7cJXnFxcZSWlrb62smTJ0lISCA6Oppdu3bx8ccfBzi7jh9DaPzBXg48p6rbpHuv7gj6kZs6a/boRCYMi+eptfu4IWM4Ib6+cTGMaV9iYiKzZ89m/PjxREVFMXhw86XuCxcu5IknnmDixImMGTOGGTNmBD5BVT3jA2d3YSWQDUQDcUBmR5Zt4/1SgawWcSiwD0gDwoFtwLiuvv/pj2nTpqkX3v40V0fev0zf/jTXk/WbL9q5c6fXKQRUa58X2KRt/FY6ustwB/AAMF1VK4AwnN2GTuvJIzd11qXjhpA2MIbH39/TWPiMCWodLQgzgd2qWiwitwIPAie7skLt4SM3dUaIT7jn4lFkHSlhXXaB1+kYc0YdLQhLgAoRmQT8G3CQFtcRmLZdPXUYg/tFsOT9vV6nYswZdbQg1Ln7Hl8Dfquqv8U5jmDOICI0hDsvGsVH+wrt1mgT9DpaEEpFZDHwdeBt98pCuwSvgxZdOIL4qDAet1aCCXIdLQg3AtU41yMcw7mQ6L/9llUvExMRym2zUvnHzjw+z2v9HLQxwaBDBcEtAi8C8e7dhVWqascQOuG2WalEh4fYsYQ+rqu3PwM88sgjVFT4d+jADhUEEbkB2ABcD9wAfCIi1/kzsd4mISacRReM4K1tuRw+YeNB9lXBXhA6eqXiT3CuQcgHEJEkYBXwmr8S643unDOKP3x0gCfW7OWXV0/wOh3jgZa3P8+fP59Bgwbx6quvUl1dzdVXX81DDz1EeXk5N9xwAzk5OdTX1/PTn/6UvLw8cnNzmTdvHgMHDmT16tV+ya+jxxB8jcXAVdiJZY1rSHwk101L4c+ZOeSXWAcqQeG5K2DLi87z+lon3vaKE9dUOHHW605cddKJd77lxOWFTrz7705cmnfG1T388MOkp6ezdetW5s+fT3Z2Nhs2bGDr1q1kZmaydu1aVqxYwdChQ9m2bRtZWVksXLiQ733vewwdOpTVq1f7rRhAx3/UK0TkHRG5zb3D8G1gud+y6sXuuTiduvoGnl1v3az1dStXrmTlypVMmTKFqVOnsmvXLrKzs5kwYQKrVq3i/vvvZ926dcTHxwcspw7tMqjqj0XkWmA2zo1OT6nqX/2aWS+VOjCGKycO5YWPD3Lf3HT6R4d7nVLfdvvbzc9Dwk6Nw6NPjSPjT41jEk+N4zo3JoeqsnjxYu65554vvJaZmcny5ctZvHgxCxYs4Gc/+1mn3rurOtzsV9XXVfVfVPWHVgzOzrfnpVNeU2/drPVBLW9/vvTSS1m6dCllZU4v3UeOHCE/P5/c3Fyio6O59dZb+dGPfsTmzZu/sKy/tNtCEJFSWu+XQABV1X5+yaqXO29IP74ydjDPf3iAuy4eRWyEDaDVV7S8/fmyyy5j0aJFzJw5E4DY2FheeOEF9uzZw49//GN8Ph9hYWEsWbIEgLvvvpvLLruM5ORkvx1HkL5wF15GRoZu2rTJ6zROseVQEVc//iGLLzuPey5J9zqdPuOzzz5j7NixXqcRMK19XhHJVNWM1ua3MwUemTIigdmjE3l63X6qatvvgtuYQLGC4KHvzBtNQVk1r246fOaZjQkAKwgemjkqkWkjE3hyzT5q67/QT6zxk76wmwxd+5xWEDwkInx33miOFFfy1y3WGWsgREZGUlhY2OuLgrrDwUdGdm7okh55eFtE5gC34OR/vqrO8jilLps7JolxQ/ux5P29XDs1xTpj9bOUlBRycnI4fvy416n4XWRkJCkpKZ1aJuAFQUSWAlcC+ao6vsX0hcBvcUZrekZV2+zmXVXXAetE5Cpgo59T9qvGVsJ9L25m2ae5fG3yMK9T6tXCwsJOGRjFnMqLXYbngYUtJ7QYyu0y4HzgZhE5X0QmiMiy0x6DWiy6CHgpUIn7y6XjhjB6UCyPr95LQ0Pvbsqa4NZTh3JDREYAJ9UZQKZH8/mE78xLZ3deKas+O/MNMsb4S7AcVGxtKLcztZ3vwBkvolUicreIbBKRTT1hf/GrE4cyYkA0j622LtuNd4KlIHR6KDdV/bmqftjO654O5dZZoSE+vj03nU9zTrLWumw3HgmWgtDrhnLrimumppAcH8nv3s22VoLxRLAUhI3AOSKSJiLhwE3AWx7nFHDhoT7uvSSdTQeL+GT/6YdZjPG/gBeEvjSUW1fcOH04A2Mj+N172V6nYvqggF+HoKo3tzF9OdYLE5FhIdx9cRr/d/kuMg8WMW1kgtcpmT4kWHYZTAu3XDiShOgwHrNWggkwKwhBKCYilDsuSmP17uNsz+nSmLrGdIkVhCD1jVmp9IsM5bHV1kowgWMFIUj1iwzjtlmpvLMjj2wb/s0EiBWEIHbb7DSiwkJYssaGfzOBYQUhiA2ICWfRhSN4c6sN/2YCwwpCkLtrzihCRKyVYALCCkKQGxIfyY3Th/PnTYfJKbJWgvEvKwg9wH1z0xGEx20oeeNnVhB6gKH9o6yVYALCCkIP8e15TivhCTuWYPzICkIPkRwfxXUZKby6MYdjJ20oeeMfVhB6kPsuSadB1VoJxm+sIPQgwwdEc83UYby04RDHS6u9Tsf0QlYQepj75o6mtr6BZ9bv8zoV0wtZQehh0gbGcMXEobzw0UGKK2q8Tsf0MlYQeqBvz02nvKae5z884HUqppcJ+oIgIqNE5FkRea3FtBgR+YOIPC0it3iZnxfGJvfjK2MHsXT9fkqqar1Ox/Qifi0IIrJURPJFJOu06QtFZLeI7BGRB9p7D3fwljtOm3wN8Jqq3gX8Uzen3SP84CvnUlJVx7Pr9nudiulF/N1CeJ7uG7atpRSaB3ap91PuQW38sHguHTeYpev327EE0238WhC6a9i2VuTgFAXoAbs9/vLD+edSVlPHE2vsjIPpHl78mDo1bJuIJIrIE8AUEVnsTv4LcK2ILAH+1sZyPWoot644b0g/rpmSwrPr97H3eJnX6ZhewIuC0Klh21S1UFXvVdV0Vf2VO61cVW9X1ftU9cU2lutRQ7l11QOXnUdkWAg/f3OHjfZkzpoXBcGGbetGSXER/GjBGNbvKeCVjYfPvIAx7fCiINiwbd3s1hkjmXPOQB58I4u1n/fO3SMTGP4+7WjDtgVAiE94/JapjB4Uy30vZPLxvkKvUzI9lPSF/c6MjAzdtGmT12n4XV5JFbc88wmHCiv4zY2TuHLiUK9TMkFIRDJVNaO11/rsKbveaHC/SF67dyYTU+L57p+28OSavXag0XSKFYRepn90OC/ceSFXTEzmV3/fxfdf3sqhQut2zXRMwEd/Nv4XGRbC726awuikWJas2cvy7Uc5LzmOIf0iuSFjOPPPH4xIa2d/TV9nxxB6ubySKpZ+sJ/Pj5XyeV4ZR4ormZQSz7hh8Zw3JI5rpqYQ6hPe25XPyMRoxg2N9zpl42ftHUOwgtCH1NU38Mqmw7y04RBHiiopqqglPiqMEJ9woty5H+Ki0QO5c04aF5+ThM/3xVbEqp15PLFmL2kDY5g1OpHRSXGkJcUQG2GNzZ7CCoIVhFZtOVTEM+v3o6rcNH0EO3JLWPrBfo6XVjNqYAxXTExm6sgEVJWcoko+3FPIih3HGD4gipMVtZRU1QEg4nTcsnDcEO6dm05EqI+DhRWkJ8US4hMaGpSa+gYiw0I8/sRfpKpsPFBEcUUNMRGhzByV2Goh7AkOFpYTHuojOT6q3fmsIFhB6LCaugbe3p7LKxsPs2H/CRpafD0SosO4dcZI/vlL5+AT2HO8jAMFFXyeV8rmQ0Ws+fw4cRGh1NQ3UFXbwIgB0cxKT+S9Xfnkl1YTEx7CiMQYxibHMX5oPJOG92diSjxhIYE7tl1YVs2WQ8VEhPkYm9yPn72ZxfLtx5peP29IHNdNS6G2XsktruRAYTmxEaGkJETh8wkx4aHMGzOI8cP6UVXbgM8HEaHNha6q1rn5tmXxq6qtJzuvjMLyahpUmTI8gYSY8KbXGxqUfQXlbDtcTH5pNQVl1RwpqqS8po6E6HAqaurZV1BGQ4MSFR7adOZoaP8oBveLICoslG05xWQeLOLui0fx75ePbXcbWEGwgtAlJ8pr2He8jNAQH4PiIkiOj2z3YGTWkZM8tXYfA2LCGT0olje2HOHTIyeZNyaJiSn9KSirZt/xcnYeLWnqJDY2IpQJw+IJD/UxtH8UC8YNZnBcJCfKazh6spK8kirKqusJDxGmjEhgWEIUpVV1ZOeVsutYKcnxkSREh/O3T3PZeqiY6roG+keHMX5YPANjwwkN8VFWVUdBWTX7C8o5eloX9j6Bf10whkvOTSI7v5RH393D/oJyAOIiQklLiqGsuo7c4kpUoaa+AVWnVdT40wkLEXwiNKhSW6/4BMYM6cfgfhEcLa5iX0EZtfV6yjpHD4olMiyEipp6cosrqahpvos/KiyEof0jiYsM40R5DRGhPtKTYokI81FeXY9PoEGVI8VVHC+toqKmnuEJ0Vw1ZRhXTRlqLYQzsYLgHVVttYjklVSx+WAR6/YUsOtoCfUNyt7j5ZRV131h3vAQH3UNDae0VgAiw3xU1TYAMKx/FF86bxAxEaHkl1ax40gJJytrqWtoICYilAEx4aQlxnDukDimjkigvKaOzANFXHxuEhekDWh6z/oGpbiihqjwEKLCQr6Qe1F5Df/4LI9DhRVER4SgCmXVdTSo4hMhNiKU6tp6Nh8qpriyhiH9ohg9KJZJKfEM6hdBXb3y4d5Cdh4toba+gcjQEIb2j2LMkFimjkggJSGaqHD/7lpZQbCC0CNU1dbz8b5CKmvqGRATzpD4SAb3i3T/J61j6+FiCstqiI0IJXVgDKmJ0RRV1JJXUsWYwXE9dt8/0NorCHZo2ASNyLAQ5o5pvZOs6PBQZqUP/ML0ATHhDGixP27Ojl2paIxpYgXBGNPECoIxpokVBGNMkz5xlkFEjgMH23h5IFAQwHTOxPJpn+XTvo7kM1JVW+1otE8UhPaIyKa2TsF4wfJpn+XTvrPNx3YZjDFNrCAYY5pYQYCnvE7gNJZP+yyf9p1VPn3+GIIxppm1EIwxTfp0QejMsPR+Wv9wEVktIp+JyA4R+b47fYCI/ENEst1/EwKYU4iIbBGRZW6cJiKfuLm84g6uE6hc+ovIayKyy91GMz3eNj90/05ZIvKSiEQGcvuIyFIRyReRrBbTWt0e4njU/W5/KiJTO7KOPlsQ2hqWPsBp1AH/qqpjgRnAd9wcHgDeVdVzgHfdOFC+jzOATqP/BP6/m0sRcEcAc/ktsEJVzwMmuXl5sm1EZBjwPSBDVccDITijjgVy+zwPLDxtWlvb4zLgHPdxN7CkQ2tQ1T75AGYC77SIFwOLPc7pTWA+sBtIdqclA7sDtP4U90v1JWAZzsC8BUBoa9vMz7n0A/bjHudqMd2rbdM4avkAnLuElwGXBnr7AKlA1pm2B/AkcHNr87X36LMtBDo5LL2/iUgqMAX4BBisqkcB3H9bvye4+z0C/BvQ4MaJQLE6w+9BYLfRKOA48Jy7C/OMiMTg0bZR1SPAr4FDwFHgJJCJd9unUVvbo0vf775cEDo1LL0/iUgs8DrwA1Ut8SiHK4F8Vc1sObmVWQO1jUKBqcASVZ0ClBPYXadTuPvmXwPSgKFADE6z/HTBctquS3+7vlwQgmJYehEJwykGL6rqX9zJeSKS7L6eDOQHIJXZwD+JyAHgZZzdhkeA/iLS2JFOILdRDpCjqp+48Ws4BcKLbQPwFWC/qh5X1VrgL8AsvNs+jdraHl36fvflguD5sPTidNj3LPCZqv6mxUtvAd90n38T59iCX6nqYlVNUdVUnG3xnqreAqwGrgtkLm4+x4DDIjLGnfRlYCcebBvXIWCGiES7f7fGfDzZPi20tT3eAr7hnm2YAZxs3LVoVyAOyATrA7gc+BzYC/zEg/VfhNOM+xTY6j4ux9l3fxfIdv8dEOC85gLL3OejgA3AHuDPQEQA85gMbHK3zxtAgpfbBngI2AVkAf8DRARy+wAv4Ry/qMVpAdzR1vbA2WX4vfvd3o5zduSM67ArFY0xTfryLoMx5jRWEIwxTawgGGOaWEEwxjSxgmCMaWIFwXhOROY23l1pvGUFwRjTxAqC6TARuVVENojIVhF50u07oUxE/p+IbBaRd0UkyZ13soh87N6L/9cW9+mPFpFVIrLNXSbdffvYFn0fvOheDYiIPCwiO933+bVHH73PsIJgOkRExgI3ArNVdTJQD9yCc5PPZlWdCqwBfu4u8kfgflWdiHOlXOP0F4Hfq+oknHsBGi+nnQL8AKdvilHAbBEZAFwNjHPf5z/8+ymNFQTTUV8GpgEbRWSrG4/CuVX6FXeeF4CLRCQe6K+qa9zpfwAuFpE4YJiq/hVAVatUtcKdZ4Oq5qhqA84l3KlACVAFPCMi1wCN8xo/sYJgOkqAP6jqZPcxRlV/0cp87V0L39otuY2qWzyvx+l0pA64AOdu0KuAFZ3M2XSSFQTTUe8C14nIIGjqy28kzneo8W6/RcB6VT0JFInIHHf614E16vT1kCMiV7nvESEi0W2t0O0nIl5Vl+PsTkz2xwczzULPPIsxoKo7ReRBYKWI+HDuuPsOTscl40QkE6cXoRvdRb4JPOH+4PcBt7vTvw48KSL/x32P69tZbRzwpohE4rQuftjNH8ucxu52NGdFRMpUNdbrPEz3sF0GY0wTayEYY5pYC8EY08QKgjGmiRUEY0wTKwjGmCZWEIwxTawgGGOa/C9W0X5cbzomOwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 18.806026\n"
     ]
    }
   ],
   "source": [
    "fit_and_plot(lambd=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yWGIF9oHSfKn"
   },
   "source": [
    "### 3.12.3.5 使用权重衰减"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:43:14.765089Z",
     "start_time": "2020-04-12T05:43:10.121500Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 215
    },
    "colab_type": "code",
    "id": "EuQKKlUbtKJd",
    "outputId": "f7a82b55-9b47-4425-b987-0c169e9ae2a8"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAC1CAYAAACQyynAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAcEUlEQVR4nO2deZxU1Zn3v09vNL1Cd7O0YlhcEZFFcDAmGR0jgmiC0aAijjEE47yOmmRilDdq4rxO1Ew06MRoMBKNJqiDu6IgBpdk3IDgiCy2CkojOzZ7N91dz/vHudVV3VZB7beoer6fT326nnPvOeep2/Wrc+6555xHVBXDMPKPAr8dMAzDH0z8hpGnmPgNI08x8RtGnmLiN4w8xcRvGHlKkd8OpIO6ujodMGCA324Yhu8sXrx4i6r2inQsp8QvImcDZx9xxBEsWrTIb3cMw3dE5JNox3Kq26+qz6rqZdXV1X67YhhZT06J3zCM2DHxG0aeklP3/DETaIe2Zigp99sTI820trbS2NhIc3Oz366kldLSUvr160dxcXHMefJP/K3N8JvRcPy34bQb/fbGSDONjY1UVlYyYMAARMRvd9KCqrJ161YaGxsZOHBgzPmyvtsvIhNF5D4ReVpExiZdYHEpjJgCXzopBd4Z2U5zczO1tbU5K3wAEaG2tjbu3o0v4heRWSKySUSWdUkfJyKrRORDEbkOQFWfUtVpwHeA81PiwCnXwpGnp6QoI/vJZeEHSeQz+tXyPwCMC08QkULgbmA8cCxwoYgcG3bK9d7x1LD3c1j9WsqKM4xINDU18dvf/jbufGeeeSZNTU1p8CiEL+JX1deAbV2STwQ+VNWPVXUf8AjwTXHcBrygqkuilSkil4nIIhFZtHnz5gM7seDn8OcLYN/uhD+HYRyIaOJvb2/fb765c+fSo0ePdLkFZNc9/6HA2jC70Uu7Evg6cJ6IXB4ts6rOVNVRqjqqV6+Isxk7c9KVcOnzUFyWnNeGsR+uu+46PvroI4YPH87o0aM59dRTmTx5MkOHDgVg4sSJnHDCCQwZMoSZM2d25BswYABbtmxhzZo1DB48mGnTpjFkyBDGjh3L3r17U+JbNo32R7ppUVW9C7grpgLCpvcekLoYzjFyipuefZ/ln+1IaZnHHlLFz84eEvX4rbfeyrJly1i6dCmvvPIKEyZMYNmyZR2j8rNmzaKmpoa9e/cyevRozj33XGprazuV0dDQwOzZs7nvvvuYNGkSjz/+OFOmTEna92xq+RuBw8LsfsBnaa1xx3p46UbYvi6t1RhGkBNPPLHT47i77rqLYcOGMWbMGNauXUtDQ8MX8gwcOJDhw4cDcMIJJ7BmzZqU+JJNLf87wJEiMhBYB1wATI6nAFV9Fnh21KhR02LK0LYX3rgb6odD9bfi9dc4yNhfC50pystDE8teeeUVFixYwBtvvEFZWRmnnHJKxMd13bp163hfWFiYsm6/X4/6ZgNvAEeLSKOITFXVNuBfgXnACuAxVX0/znLPFpGZ27dvjy1DzSC45kM4zoRvpIfKykp27twZ8dj27dvp2bMnZWVlrFy5kjfffDOjvvnS8qvqhVHS5wJzkyg3vpYfoHvPRKszjANSW1vLySefzHHHHUf37t3p06dPx7Fx48Zx7733cvzxx3P00UczZsyYjPomubRvf9iA37RI904RUYUnL4eKXjD25rT6Z2SeFStWMHjwYL/dyAiRPquILFbVUZHOz6YBv6RJaD2/CHSrgGJb5GPkF9k04OcfE2732wPDyDg51fLHPeDXlR3pfbJoGNlETok/qW28/nYn3DkM9nSddWwYuYl1+4MceQYUlkCBXRIjP8ipb3pc03u70vsY9zKMPMG6/eG0t8KqF6Hp09Q6ZuQtiS7pBZgxYwZ79uxJsUchckr8SbN7CzxyISyd7bcnRo6QzeLPqW5/0lTVw6UvwqEj/fbEyBHCl/Sefvrp9O7dm8cee4yWlhbOOeccbrrpJnbv3s2kSZNobGykvb2dG264gY0bN/LZZ59x6qmnUldXx8KFC1PvnKrmzAs4G5h5xBFHqGGoqi5fvrxzwqwzVZc87N637XP20kec3bLb2e/NcfbeJme//7Szd21x9sq5zt6x4YD1r169WocMGaKqqvPmzdNp06ZpIBDQ9vZ2nTBhgr766qs6Z84c/d73vteRp6mpSVVV+/fvr5s3b078s6oqsEij6CWnuv2aqog9ix+E1/4zNU4Zhsf8+fOZP38+I0aMYOTIkaxcuZKGhgaGDh3KggULuPbaa3n99dfJVMQp6/ZHovFt2LYGvvpjN/3XyB0ufT70vrC4s11S1tkure5sl9d2titDi3RiQVWZPn063//+979wbPHixcydO5fp06czduxYbrwx/dvKm/gjMeEOKOp24PMM4wCEL+k944wzuOGGG7jooouoqKhg3bp1FBcX09bWRk1NDVOmTKGiooIHHnigU966urq0+Gbij0RQ+IEAFOTUnZGRYcKX9I4fP57Jkydz0kkuZkRFRQUPP/wwH374Iddccw0FBQUUFxdzzz33AHDZZZcxfvx46uvr0zLgZ0t6o9GwAJ7+PzBtIVQfmhL/jMxjS3ptSW/81A6Cw/4BWlOzZZJhZBvW7Y9GzSA4/yG/vTCMtJFTLX9a2L3F7fJrGDmGiX9/tDbDncPhddvs42Aml8a1opHIZ7Ru//4oLoWz7oC+x/vtiZEgpaWlbN26Nacj9aoXoru0tDSufFkvfhEZBPwUqFbV8zLuwPGTMl6lkTr69etHY2MjMcVvPIgpLS2lX79+ceXxRfwiMgs4C9ikqseFpY8D7gQKgd+r6q2q+jEwVUTm+OErABuWQeM7MOpS31wwEqO4uLhThBwjxMEUots/3vtvmPdTaIkcfMEwDkayPkR3rGXGGqK7ac8+nvx7I581xfH8/stXwo/eh26VsecxjCwnm0b7I4boFpFaEbkXGCEi06Nl1hhDdK/f3swPH32Xd9c2xe5ZeZ1F9jFyjmwSf7QQ3VtV9XJVPVxVb9lvATFs3V3dvRiApr2t8Xm3bTX8+Xz47O/x5TOMLCWbxJ+REN1B8W+PV/zde8Lmlba3v5EzZJP4O0J0i0gJLkT3M/EUEMvc/rKSQooKJAHx94CrlsIxE+LLZxhZSt6F6BYRqrsXxy9+l9kF9mzeEX9ew8gy8jJEd8LiB3ffH2iFi59MLL9hZAlZP8MvHmIN2lHVvZgdiYp/8NmggcTyGkYWkU33/EkT63r+pFr+kRfDCZckltcwsoicEn+sJCV+gLYWWPGs2+bLMA5Sckr8sYbo7lGWpPhXPg+PToE1rydehmH4TE6JP55u/469rQQCCa7zPmYCTHkCBnwlsfyGkQXklPhjpbp7MQGFXfvaEiugqBsccRoUFKbWMcPIIDkl/li7/VXBWX57kuj6BwLw1xmw9M+Jl2EYPpJT4o+n2w8JTPENp6AAVr0Aq+2+3zg4yann/LGSEvGDm+hTUpYCjwwj8+RUyx8rKRN/UPjtSZZjGD6QU+KP9Z4/ZeIH97z/9mNg54bkyzKMDJJT4s/oPX+Q3sfCoFPcxB/DOIjIy3v+hJf1RqL2cDjv/uTLMYwME1PLLyJXi0iVOO4XkSUiMjbdzqWLpJb1RmP7OtjwXurKM4w0E2u3/7uqugMYC/QCLgVuTZtXGSCl4leFh86B53+cmvIMIwPE2u0P7q93JvAHVX1XsjD8SaxLeiHJZb1frBjOngFVFsrbOHiIteVfLCLzceKfJyKVQNYtaYsnRHfKu/39vww9+6euPMNIM7GKfypwHTBaVfcAxbiu/0FL0iv7IrF9HTx5OWxpSG25hpEGYhX/ScAqVW0SkSnA9cD+H6ZnOSlv+QEKS+CDebAxrq0HDcMXYhX/PcAeERkG/AT4BPhj2rzKAEkv641ERS/40XIYMjF1ZRpGmohV/G3qAoB/E7hTVe8EDurYVcFlvTtbElzWG43i7u7v9sbUlmsYKSZW8e/0QmVdDDzvBdUsTp9bIUSkXEQeFJH7ROSiVJVbnYplvdFY9Ae4c5iL8mMYWUqs4j8faME979+Ai6v3n4lWKiKzRGSTiCzrkj5ORFaJyIcicp2X/C1gjqpOA76RaJ1dqSkvAWDr7jRMyz1qHHztGqjonfqyDSNFxCR+T/B/AqpF5CygWVWTued/gNhDdPcjFMCzPYk6OxEU/7bd+1JVZIiqejjlOigpT33ZhpEiYp3eOwl4G/g2MAl4S0TOS7TSOEN0N+J+APbrb6whuoPUlncDYGs6xB+kcTE8dQUEUvabZRgpI9Zu/09xz/gvUdV/xgn1hhT7EjFEN/AEcK6I3AM8Gy1zrCG6g9RUpLHlD7LtY/joL7B97YHPNYwME+v03gJV3RRmbyX1y4GjhejeTYwTiuKZ3lteUkhJUUF6xT/0PDjqDCitSl8dhpEgsQr4RRGZJyLfEZHvAM+TREy9KGQkRHcQEaG2vIStu9IofhEnfFV4b46t+TeyilgH/K4BZgLHA8OAmap6bYp9yUiI7nBqykvYlo7R/q588jd4fCq8Ozv9dRlGjMS8mYeqPg48nopKvRDdpwB1ItII/ExV7xeRYIjuQmBWIiG6ibHbD0Hxp7HlDzLgK/CduW7xj2FkCfsVv4jsBCLNfxXc/XhCN7N+h+gOUltewuotuxOtLj4GnOz+7tzgbgFOusLdFhiGT+y326+qlapaFeFVmajw00msG3gGqSnvlpmWP5y/PwQLfwG7Nh34XMNII3m5gWeQ2ooS9uxrp7k1g8/hv/pjmDoPKvs4+4N5tvW34Qs5Jf54Sessv2iIQN+h7v26JfDnSbDEmyzZ1gJtGe6JGA5VaG8LhV1XhdZmlxa0W3aF/j+BAOxtCj3BCbTDnm0uD7h8uzaF2a2w4zNo3evsthZoWhuyW5vh8zUhe98e2PpRmL0bNn8Qslt2Jt1o5JT44+/2+yD+cA4ZAZMfg2EXOHv5M3BLP2j61Nlr34ZXfxn6h29Y5mIDBmcMfrYUFj8YKu+T/4E37g7ZH8yHV24L2cuegJd+FrKXPAQvhD20eet38NyPQvZffw3PXBWyF/4Cnv7XkD3/Bnj6ipD9/I87H3/qis72nO/CM1eG7NkXwrNXh+w/ToTnfhiy7z+j876I9361s7//NQpe/L8h+44hMP/6kH3bwM6f9+Y+8Jeb3ftAAH5eDa94W1G2tcD/q4W/zXB2yw74jz7w1r3O3rMVbjkUlnjXe9cGuK1/6AlO06fwy4Hw/hPO3vYx/OpIWPmcszevhDsGQ8NLzt64DGYcB6tfc/b6pW4x2KdvOLvxHfivka6BAPjkDbh7tPsOgMu3ayPJkFPij7vb37G4xyfxi7hJQME1AL2OghOnQYV3S/Dpm7DwP0C91uiDF+CpfwmJf9VcePYq1yoBNMyHBT8Plb/6VXgz7Mdg/VJYEfb0dNtH0LgoZO/cAJ+HrURs3gF7Pw/ZgfbOU5ULi6GwW8gurYLSsGtf2Sf0WQB6DoQeXwrZvQdDbdiTmUNHQq/BIXvAyaFeEsCRY6F+eMg+9hvQ74SQPewCOOwfQvboqZ2fsHz5qpAtAv94LfT3BmILiuDU60PHi0rhtBvhSyc5u7gMTv93OOxEZ3ergjN+Af1GO7usFsb/Eg71/KnoBRNudz/w4PZ3PGsG1B/v7B794Ru/gT5DnF1zOEy8J/T5ex0D58yEuiOd3fc4OPd+qBnk7ENGQPcakkFUU7iZRZYwatQoXbRo0QHP+3jzLv7p9lf59fnDOGdEvwOe7wtt+5zIRFw3c+/n0HOAs5u3u65o1SHObm2GQBt0q/DbayNLEJHFqjoq0rGcCtoR73P+jsU96ZzllyxFJaH33Xu4V5DS6s4tbXFp5vwyDnryuttf1b2IogLx757fMHwkp8QfLyJCz0zN8jOMLCOnxB/vaD+4QT/fBvwMw0dySvzxdvshg/P7DSPLyCnxJ4KJ38hX8l78dRXd2LzT1tkb+Ufei7++upRdLW3sbLb59UZ+kffi71vtno1v3NHssyeGkVlySvyJjPbXV7sIO+u3m/iN/CKnxJ/IaH/fKtfym/iNfCOnxJ8IvavcFN+NJn4jz8h78ZcWF1JbXsJ6u+c38oy8Fz+4Qb8N1vIbeUbWi19EBonI/SIyJ1111FeX2j2/kXekVfxxRuONiBe7b2o6/exTVWqP+oy8I93r+R8AfgN0RPQNi8Z7Oi5Kzzsi8gxur/5buuT/bpcwYWmhvrqUbbv30dzaTmlxYbqrM4ysIK3iV9XXRGRAl+SOaLwAIvII8E1VvQU4K53+RKOv96x/445m+tdaWG0jP/Djnj9aNN6IiEitiNwLjBCR6fs5L64Q3eHUV9uzfiP/8GMbr4jReKOdrKpbgcsPVKiqzsTFE2TUqFFxbUzYx5voYyP+Rj7hR8uftmi8iUzvhdD8/g026GfkEX6IP+lovKmmolsRlaVF1vIbeUW6H/XNBt4AjhaRRhGZqqptQDAa7wrgsXij8UYjkbn9QeqrS1nXtDcVbhjGQUG6R/vTEo03GvFu3R3OoLoKPti0M9UuGUbWkvUz/OIhmZb/qL6VrNmyO7NBOw3DR3JK/IkO+AEc1aeCgMJHm3elwTPDyD5ySvzJtPxH96kE4ION1vU38oOcEn8yDKgrp7hQWLXBWn4jP8gp8SfT7S8uLODwXhXW8ht5Q06JP5luP8BRfSpZtcHEb+QHOSX+ZDm6byXrmvayq6XNb1cMI+3klPiT6faDa/kBGqzrb+QBOSX+ZLv9wRF/6/ob+UBOiT9Z+vXsTs+yYt78eKvfrhhG2skp8Sfb7S8oEMYe25cFKzbZTD8j58kp8Sfb7QcYP7Qvu1ra+GvDlhR6ZhjZR06JPxWcfEQd1d2Lmbtsvd+uGEZaMfF3obiwgNOP7cNLyzfS0mZdfyN3MfFHYMLQenY2t/HkknV+u2IYacPEH4F/PKoXYwbVcPPzK1i7bY/f7hhGWsgp8Sc72h+koED41beHIcAPHl3KjubW1DhoGFlETok/FaP9Qfr1LOMX3xrK0rVNnPHr13hx2Xr2tQVS4KVhZAd+bN190HD2sEM4rKaMf3tsKZc/vISq0iJOHFjLUX0qqO/RnbryEipKiygrKaK0uICSwgKKCwsoKhSKCtzfQhEKg38LhIKOvyASaRdzw8gMJv4DMPywHrxw9dd4vWEzLyzbwLtrm1i4ahPtgbhCA0SkQOj0g1AoQoH3wxBMDx4TL61QQu+DxwsKCL0X731Hec6W8GPeD0+wHvHyCnTUJYSXRZf8Xc4pcHmlo+xQnUJ4ne6cruV3pHvXJFgOEiyXDv8KXOYv1Cd0LqdTGqG6C8KPe+cEI0kEy++aly+U1bncYAGdjnfxIUhnXzsfP1B5dBx3+YoKhUN6dE/4+2fij4GSogJOG9yH0wb3AaC1PcDnu/exZdc+drW0sXtfG/vaArS0BWhrD9DWrrQGAgQCSltAaQ++VAkElPYAofcdae69KmHvXXpA6Ti3PeDOCYQfU+2wgVB9AWVfu1eOAtq5jo5yVKGjHDrKCZbpsnr5vL8BLz1YtyruhStDvbKM9PGlmjJe+8mpCefPevGLyERgAtAbuFtV5/vsEsWFBfSuKqW3F+nHiE7wRyD4AxFQ9dLdD0XwBwzPDp4T/GFx6Z3ze0V0Ojf0Y+P9EIXVEfxh6viR6nSO50/w/I7yurwPOgLuxy2sjODn9KrvVGekOrr66KV8obzw60T4ud555SXJBZVNq/hFZBYu+OYmVT0uLH0ccCcuMu/vVfXWaGWo6lPAUyLSE/gV4Lv4jdgREQoFCiNGaTP85GAK0X29l88wjBSQ9SG6xQ2J3wq8oKpL0umvYeQTWR+iG7gS+DpwnohEjdabTIhuw8hHDoYQ3XcBdx2o0GRCdBtGPuKH+NMaohs4G9ghIg1RTqsDsmmxvvkTnWzyBQ5Of/pHOyDBRwrpwrvnfy442i8iRcAHwGnAOlzI7smpitQbgz+LVHVUJuqKBfMnOtnkC+SePzkVotswjNjJqRDdhmHETk6t6ouRmX470AXzJzrZ5AvkmD9pv+c3DCM7yceW3zAM8kj8IjJORFaJyIcicp0P9R8mIgtFZIWIvC8iV3vpNSLykog0eH97ZtivQhH5u4g859kDReQtz59HRaQkg770EJE5IrLSu04n+Xl9ROSH3v9qmYjMFpHSTF4fEZklIptEZFlYWsTrIY67vO/3/4rIyAOVnxfiD1tPMB44FrhQRI7NsBttwL+p6mBgDHCF58N1wMuqeiTwsmdnkqtxT12C3Ab82vPnc2BqBn25E3hRVY8Bhnl++XJ9RORQ4CpglPeYuhC4gMxenweAcV3Sol2P8cCR3usy4J4Dlq7euvFcfgEnAfPC7OnAdJ99ehq3uGkVUO+l1QOrMuhDP+8L9E/Ac7jZl1uAokjXLc2+VAGr8cahwtJ9uT6EpqHX4J6KPQeckenrAwwAlh3oegC/Ay6MdF60V160/MS/niCteBOfRgBvAX1UdT2A97d3Bl2ZAfwECG5OWAs0qZuLAZm9ToOAzcAfvNuQ34tIOT5dH1Vdh1tC/imwHtgOLMa/6xMk2vWI+zueL+KPaz1BOhGRCuBx4AequsMPHzw/gvssLA5PjnBqpq5TETASuEdVRwC7yfwtUAfevfQ3gYHAIUA5rmvdlWx5XBb3/y5fxJ+29QTxICLFOOH/SVWf8JI3iki9d7we2BQtf4o5GfiGiKwBHsF1/WcAPbwp2JDZ69QINKrqW549B/dj4Nf1+TqwWlU3q2or8ATwZfy7PkGiXY+4v+P5Iv53gCO9kdoS3MDNM5l0wNuX4H5ghareEXboGeAS7/0luLGAtKOq01W1n6oOwF2Pv6jqRcBC4Dwf/NkArBWRo72k04Dl+HR9cN39MSJS5v3vgv74cn3CiHY9ngH+2Rv1HwNsD94eRCUTgyfZ8ALOxC0o+gj4qQ/1fwXXDftfYKn3OhN3n/0y0OD9rfHBt1Nwi6/A3Xu/DXwI/DfQLYN+DAcWedfoKaCnn9cHuAlYCSwDHgK6ZfL6ALNx4w2tuJZ9arTrgev23+19v9/DPaXYb/k2w88w8pR86fYbhtEFE79h5CkmfsPIU0z8hpGnmPgNI08x8RsZRUROCa4gNPzFxG8YeYqJ34iIiEwRkbdFZKmI/M5b979LRG4XkSUi8rKI9PLOHS4ib3rryJ8MW2N+hIgsEJF3vTyHe8VXhK3b/5M3gw4RuVVElnvl/Mqnj543mPiNLyAig4HzgZNVdTjQDlyEW9yyRFVHAq8CP/Oy/BG4VlWPx80uC6b/CRdZeRhuXnxwuukI4Ae4vRUGASeLSA1wDjDEK+fm9H5Kw8RvROI04ARcENWlnj0It/T3Ue+ch4GviEg10ENVX/XSHwS+JiKVwKGq+iSAqjar6h7vnLdVtVFVA7hpzgOAHUAz8HsR+RYQPNdIEyZ+IxICPKiqw73X0ar68wjn7W9u+P5icreEvW/HbY7Rhgvi+jgwEXgxTp+NODHxG5F4GRcYtTd07BvXH/d9Ca5omwz8VVW3A5+LyFe99IuBV9XtVdAoIhO9MrqJSFm0Cr19DqrVxXT4AW6Rj5FG/IjVZ2Q5qrpcRK4H5otIAW5V2RW4DTaGiMhi3M4253tZLgHu9cT9MXCpl34x8DsR+XevjG/vp9pK4GkRKcX1Gn6Y4o9ldMFW9RkxIyK7VLXCbz+M1GDdfsPIU6zlN4w8xVp+w8hTTPyGkaeY+A0jTzHxG0aeYuI3jDzFxG8Yecr/BxkTd11kU2wMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 0.25075185\n"
     ]
    }
   ],
   "source": [
    "fit_and_plot(lambd=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xJiYKczaS3PS"
   },
   "source": [
    "## 3.12.4 简洁实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:43:50.133783Z",
     "start_time": "2020-04-12T05:43:50.127798Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "mJrspxkzSvAM"
   },
   "outputs": [],
   "source": [
    "def fit_and_plot_tf2(wd, lr=1e-3):\n",
    "    model = tf.keras.models.Sequential()\n",
    "    model.add(tf.keras.layers.Dense(1, \n",
    "                                    kernel_regularizer=regularizers.l2(wd), \n",
    "                                    bias_regularizer=None))\n",
    "    model.compile(optimizer=tf.keras.optimizers.SGD(lr=lr), \n",
    "                 loss=tf.keras.losses.MeanSquaredError())\n",
    "    history = model.fit(train_features, train_labels, epochs=100, batch_size=1, \n",
    "              validation_data=(test_features, test_labels),\n",
    "             validation_freq=1,verbose=0)\n",
    "    # model.summary()\n",
    "    train_ls = history.history['loss']\n",
    "    test_ls = history.history['val_loss']\n",
    "    # print(test_ls)\n",
    "    semilogy(range(1, num_epochs + 1), train_ls, 'epochs', 'loss',\n",
    "                     range(1, num_epochs + 1), test_ls, ['train', 'test'])\n",
    "    print('L2 norm of w:', tf.norm(model.get_weights()[0]).numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:43:57.853129Z",
     "start_time": "2020-04-12T05:43:50.932646Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAC1CAYAAAC56YFuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXhV9dXo8e/KTEIImQMECCQMISBTmBFBFFF8rdUWK2ptyxX76nu19tUqtba117e1T1uvQx1wwKH6OFwciygIMskgBAgSAiEBIgQIGchExpOT3/1jnwxARsgZkqzP8+Qha589rLPPyWLv39779xNjDEopBeDl7gSUUp5DC4JSqoEWBKVUAy0ISqkGWhCUUg20ICilGvi4OwFXiIiIMHFxce5OQymPsGvXrgJjTGRzr/WIghAXF0dKSoq701DKI4jI9y29pqcMSqkGWhCUUg20ICilGmhBqLfp77D1n43x2sdhyzON8ZrHYNsLjfGXv4VvX26MV/0Gdr7WGK/8Nex6szH+7D7Y805j/Mk98N0HjfFHSyDtQ+t3Y2DFYkj/1IrtNis++LkV2yqt+NBqK64us+KstVZcWWTFRzZY8dl8K87eYsWlp6z42LdWXHzcinN2WfGZo1Z8MtWKC7KsODfNivMOWHF+hhXn7rPiwsNWfDLVis8cteKcFCsuybHiY9utuCzXirO/seLyAis+ssGKK4usOGutFVeXWXHGl1Zsq7LiAyut2F5rxfs/seJ6+1bAR3c3xnvfh0/ubYz3vGN9PvV2vWl9fvV2vgpfPNwYf7sMVj/aGG97Hr76fWO85RlY96fGePM/YP2fG+MNf4WNf2uMv/4f2PxUY7zuT5f23bsEPaJRsS3LvznKqM1rmDJyEFI/MT8Dekc1zpR/EGqrGuO89HNXcno/+PZqEqdBYFhjnLsP+gxojE99B+EJTeK9EDOmMT65BwZOtn43xooHT7fiOrsVD51txXabFQ+bd2488jpHXG3FlWesuLbKERedG1c5YluFFVeXWnHNWUfs+IOsro/POuIyK65xxFUlVmyrtOLKYkfs2H+VRVZcvz8rCq3YXmPF5QWOuPbcuK4+zrNiY7fis6cdcZ0Vl+Vacb2yU3AqtTEuzYHcvY1xyXHr86pXfMz6POsVfX9enG19H+qdOWotU6/wsPWemsa2iiZxJkiT/4sLDoFf78Y4PwOCIprEHfzuXQLpCU87Jicnm9auMry9/Xt+90kaX//3FQyN7N3ifEp1ByKyyxiT3NxresoAzEiwqvGWw4VtzKlU96YFAYgLD6R/SABbswrcnYpSbqUFARARpidEsO1IIXV13f8USqmWaEFwmJEQTnGFjfRTpe5ORSm30YLgMD3e0Y6gpw2qB9OC4BDdJ4CEqN58owVB9WBdsiCIyFAReU1EVnTmeq8cGcX2I4WUVNo6c7VKdRkuLwgislxE8kQk7bzp80UkQ0SyROSR1tZhjDlijFnc2jwXY/7oGGx2w9cHT3f2qpXqEtxxhPAGML/pBBHxBp4HrgVGAbeKyCgRGSMiK8/7ibpwlZ1jXGxfYvoE8MW+XGdtQimP5vJbl40xm0Qk7rzJk4EsY8wRABF5D/iBMeYvwPWuys3LS5g/OoZ3dxyjvLqWIH+9s1v1LJ7ShjAAON4kznFMa5aIhIvIS8B4EVnawjxLRCRFRFLy8/Pbncj80TFU19axIaP9yyjVXXhKQZBmprV4h5AxptAY80tjTLzjKKK5eV42xiQbY5IjI5vtLapZk+LCCA/yY/V+PW1QPY+nFIQcYGCTOBY46Y5EvL2EOSOj2JCRh81e544UlHIbTykIO4FhIjJERPyAnwCfuSuZqxKjKa2qJSW7yF0pKOUW7rjs+C6wDRghIjkistgYUwv8F7AaOAB8YIzZ39p6nOnyYRH4eXux7oBeflQ9izuuMtzawvRVwCoXp9OsIH8fpsWHs/bAaR5dkIhIc00cSnU/nnLK4HGuSowiu7CCw/nl7k5FKZfRgtCCuYnRAHyVrqcNqufQgtCC/n17MWZACF+l6+VH1XNoQWjFvFHR7DleTF5pVdszK9UNaEFoxbykGIyBtQfy3J2KUi6hBaEVw6N7Mzg8kDV62qB6CC0IrRAR5o2KZmtWIWVV2keC6v60ILRhbmI0NfY6tmoX7aoH0ILQhgmDQunt78PGQ/r0o+r+tCC0wc/Hi2nx4Ww6lE9PGOVK9WxaENrhiuGR5BRVcrRA71pU3ZsWhHa4YrjVn4KeNqjuTgtCOwwMC2RIRBCbtCCobk4LQjtdMTySbUcKqaipdXcqSjmNFoR2uiYphiqb9rWourcuWxBEJEhEdomIS3plnjwkjIjefny+75QrNqeUW3TJgVocHgY+cE6WF/L2Eq5JimH9wTwqa+yu2qxSLtUlB2oRkauAdMClnRVcN6YfFTV2vdqguq0uOVCLiMwBgrCKR6WIrDLGOL2L5ClDwggL8mPVvlPMHx3j7M0p5XKeMjRRcwO1TGlpZmPMowAi8jOgoLliICJLgCUAgwYN6pQkfby9uHJkFF+ln8ZeZ/D20r4WVffiKY2KHRqopWEGY94wxqxs4bWLGqilLVcMj6Sk0sbenOJOW6dSnsJTCoLHDNTSlpkJEXgJbNTLj6ob8pSC4FEDtbQmNMiPsQP7asOi6pZ0oJaLMGtYJHtziikqr3F3Kkp1KpcXBGPMrcaYfsYYX2NMrDHmNcf0VcaY4Y4BXP/H1Xl1xBUjIjEGNmcVuDsVpTqVp5wydCljY/sSGujL1zrUm+pmtCBcBG8v4cqR0Xx9UEeIVt2Lp9yH0OXMS4rmw9057Dh6hhkJEe5OR7WTzWYjJyeHqqruP9ZGQEAAsbGx+Pr6tnsZLQgXadawSAJ8vfgq/bQWhC4kJyeH4OBg4uLiuvUgvsYYCgsLycnJYciQIe1eTk8ZLlIvP29mJkSyZn+u9rXYhVRVVREeHt6tiwFYQwiEh4d3+EhIC8IlmJcUzcmSKvafLHV3KqoDunsxqHcx71MLwiWYPcK6JXrrYb38qLoHLQiXICo4gCERQezMLnJ3KqqLKC4u5oUXXujwctdddx3Fxc5/fkYLwiVKHhxKSvYZ6uq0HUG1raWCYLe33unOqlWr6Nu3r7PSaqAF4RJNigujqMLGkYKz7k5FdQGPPPIIhw8fZty4cUyaNIk5c+awaNEixowZA8CNN97IxIkTSUpK4uWXX25YLi4ujoKCArKzs0lMTOSuu+4iKSmJefPmUVlZ2Wn56WXHS5QcFwrAzuwiEqKC3ZyN6ojH/72f9E5uEB7Vvw9/+I+kFl9/8sknSUtLIzU1lQ0bNrBgwQLS0tIaLg0uX76csLAwKisrmTRpEjfffDPh4eHnrCMzM5N3332XV155hYULF/Lhhx9y++23d0r+eoRwiYZEBBEe5MfO7DPuTkV1QZMnTz7nPoFnn32WsWPHMnXqVI4fP05mZuYFywwZMoRx48YBMHHiRLKzszstn3YdIYjI/cDrQBnwKjAeeMQYs6bTMumiRITkuFBStGGxy2ntf3JXCQoKavh9w4YNrF27lm3bthEYGMjs2bObvY/A39+/4Xdvb+9OPWVo7xHCL4wxpcA8IBL4OfBkp2XRxU2KC+PYmQpOFHfeB6O6p+DgYMrKypp9raSkhNDQUAIDAzl48CDbt293cXbtLwj1dzhcB7xujNlL892euYSIXC4iL4nIqyKy1V151JubGA3AFzpmg2pDeHg4M2bMYPTo0Tz00EPnvDZ//nxqa2u57LLLeOyxx5g6darL85P23HYrIq9jdYQ6BBgLeAMbjDETO7xBkeVYPSnnGWNGN5k+H3jGse5XjTFtHoGIyI1AtDFmWWvzJScnm5SUlI6m2iELnt2Mr7cXn9w7w6nbUZfmwIEDJCYmujsNl2nu/YrILmNMcnPzt/cIYTHwCDDJGFMB+GKdNlyMN7jEcRmaLLoIePci8+hU11/Wn9TjxRw/U+HuVJS6aO0tCNOADGNMsYjcDvwOKLmYDRpjNgHnN8k3jMtgjKkB6sdl2GeMuf68nzwAERkElDjaNtxuwZh+ADrUm+rS2lsQXgQqRGQs8Bvge+CtTsyjuXEZBrSxzGKsKx/NEpElIpIiIin5+c7vEHVQeCBjY0P4/DstCKrram9BqDVWY8MPgGeMMc8AnXkXTofHZTDG/MEY02KDorPGZWjNvKQY9p0ooeBstUu2p1Rna29BKBORpcAdwOeOc/72d8PSti4zLkNrpsdbd5RtO1zo5kyUujjtLQi3ANVY9yPkYh3O/60T8+gy4zK0ZsyAEIIDfPRxaNVltasgOIrAO0CIiFwPVBljLqoNoTuMy9ASH28vpg4NZ0uWHiGo5l3s488ATz/9NBUVzr2K1a6CICILgR3Aj4GFwLci8qOL2WB3GJehNTPiwzl2pkIvP6pmeXpBaO/Tjo9i3YNQf8kvElgLrHBWYl1VfYerW7IK+Mnkzhl1WnUfTR9/vvrqq4mKiuKDDz6gurqaH/7whzz++OOUl5ezcOFCcnJysNvtPPbYY5w+fZqTJ08yZ84cIiIiWL9+vVPya28bgld9MXAo7MCyPUpCVG+igv35Rkd16hpeXwB73rF+t9useO/7VlxTYcVpH1pxVYkVpzuat8oLrTjjCysua3vgnieffJL4+HhSU1O5+uqryczMZMeOHaSmprJr1y42bdrEl19+Sf/+/dm7dy9paWnMnz+f++67j/79+7N+/XqnFQNo/x/1lyKyWkR+JiI/Az4HVjktqy5MRJg5LIItWQXYtRcl1Yo1a9awZs0axo8fz4QJEzh48CCZmZmMGTOGtWvX8vDDD7N582ZCQkJcllO7ThmMMQ+JyM3ADKx7Bl42xnzs1My6sCuGR/LR7hOknShh7EDnd3ulLsHPP2/83dv33Ngv8Nw4IOTcOCj83Dg4ukObNsawdOlS7r777gte27VrF6tWrWLp0qXMmzeP3//+9x1a98Vq92G/MeZDY8yvjTEPaDFo3cyECETQIePVBZo+/nzNNdewfPlyzp61ut87ceIEeXl5nDx5ksDAQG6//XYefPBBdu/efcGyztLqEYKIlNH8HYMCGGNMH6dk1cWF9/ZnzIAQNh3K5765w9ydjvIgTR9/vvbaa1m0aBHTpk0DoHfv3rz99ttkZWXx0EMP4eXlha+vLy+++CIAS5Ys4dprr6Vfv35Oa0do1+PPXZ0rHn8+3z/WZPD8+iz2PDaPkMDOvKlTXQp9/LlzHn9WHTRreCR1BrboXYuqC9GC4CTjB/YlOMCHDRl5bc+slIfQguAkPt5eXD4sgo2H8nUwWA/TUz6Pi3mfWhCcaPaIKE6XVnPglHNbhlX7BQQEUFhY2O2LQv1w8AEBAR1aTgdqcaLZw61+GNZn5DGqv16Q8QSxsbHk5OTgik5z3C0gIIDY2NgOLaMFwYmi+gQwekAfNmTkce+cBHenowBfX99zBkZR59JTBiebMyKK3ceKKamwuTsVpdrk8QVBRIaKyGsisqLJtCAReVNEXhGR29yZX1tmj4jCXmfYcEivNijP59SCICLLRSRPRNLOmz5fRDJEJEtEHmltHY6emBefN/kmYIUx5i7ghk5Ou1ONH9iXyGB/Vu/PdXcqSrXJ2UcIb9B5YzA0FUtjL812J+XeKby8hGuSoll/MJ8qm0enqpRzC0JnjcHQjBysogBd4LRnflI/Km12fdhJeTx3/DF1aAwGEQkXkZeA8Y6enwE+Am4WkReBf7ewnEvHZWjNlKFh9A30ZXWanjYoz+aOy44dGoPBGFMI/PK8aeW0MZScMeZl4GWwHm7qeJqdx9fbi6sSo1m9P5ea2jr8fDz+oEb1UO74ZnaLMRg6at6oaMqqakn5/vwzKKU8hzsKQrcYg6GjZiRE4OstbMzQdgTluZx92bHbjsHQUUH+PkweEsZ6ffpReTCntiEYY25tYfoqemAnrXNGRPHE5wc4UVzJgL693J2OUhfQ1i0Xmj3CethJ+0hQnkoLggvFR/YmNrQXG7QdQXkoLQguJCLMGRHFN5kFetei8khaEFxsXlI0lTY7mzO1r0XlebQguNiUIeEEB/jow07KI2lBcDE/Hy/mjoxi3YHT1Nrr3J2OUufQguAG1yTFUFRhY2d2kbtTUeocWhDcYNbwSPx8vPS0QXkcLQhuEOTvw/T4cH0cWnkcLQhuMmtYJEcLyjl+psLdqSjVQAuCm8waHgHApkw9SlCeQwuCm8RH9qZ/SACbD+n9CMpzaEFwExFh1vBIthwu0MuPymNoQXCjy4dFUlZVy96cYnenohSgBcGtZiZE4CWwSU8blIfw+ILQwkAtiSLykoisEJH/dGd+lyIk0JfRA0LYdrjQ3akoBXTRgVqMMQeMMb8EFgLJnZ+560yLD2fP8SIqamrdnYpSXXagFkTkBuAbYJ3z0ne+GfER2OxGb2NWHqGrDtSCMeYzY8x0wKPHdmxLclwovt7C1sPajqDcr0sO1CIis0XkWRFZRgt9M3rSQC2tCfTzYfzAUG1HUB6hqw7UsgHY0NpGPGmglrZMTwjnmXWZrE0/zYiYYAaGBbo7JdVD6UAtHuDKkVEYA//rrRRm/W09v1mxl/yyanenpXogdxwhNAzUApzAGqhlkRvy8BiXxfZl+9K55BRVsHp/Lm9szWZLViGr7r+ckF6+7k5P9SA6UIuHiAkJIDkujEcXjOK9JdPILa3isU/S2l5QqU6kA7V4oImDQ7l/7jCe+uoQp0urOFtdy12XD+XG8S22vSo3MsYg0lzTWNfj8Xcq9lT3zI7nmiRrgFibvY4HPkjlg5TjF8z3VfppjuSfbXN9mw7lM/0v6/gq/bQz0u0SisprWJ+RR/rJUmyOB8pKKm388bP9bM7Mx5j2tT1/l1PMqn2nAMjKO8uUP6/jH2syLjqvnKIK9h4v5mhBebtzADhZXMm+nJJO7dLfHW0Iqh18vL1Ydod1E2aVzc5db6Xw8IffUVZVy+KZQwB4e/v3/O6TNMYMCOGz/5pxwf9S1bV2MnLL2HOsmCc+T8dmN/zh0zRmJIQT6HfhR3/8TAXXP/cNlw+L4MF5I4iLCHL+G3WiujrDtiOFJMeF4uftxT3v7GbbEevy7riBfXlvyVSe/OIg7+44xhtbsxkZE0xvfx9iQgK4Z3YCo/r3uWCdx89UcMdrOyiptHHflQl8vu8UheU1PPd1FgG+3tw7J6Hd+VXW2HlmXSavbD6Cvc4qBEMjgrg6KZq9x4vJyC3DXmcIDfIjqX8ffj5jCJPiwtiaVcBvP95HdqHVuY6vtzB1aDh3TB3M3MRovL0u/mhFOlKRuqrk5GSTkpLi7jQuSZXNzgPvp/JFWi43jO1PaKAvb23/npg+AZwqqeK9JVOZOjT8nPkXLtvGdzklAEyOC+PuK4ay+M0U7rsygV/PG3HBNl7dfIQnPj9AL19v7HWGF26bwFWjosktqeLLtFNsOVzITyYNZG5itMved71aex3eXtLuQ/Mqm51ff5DKqn25zE+K4doxMdz/Xir3zx1Gn16+/J+V6UwbGs62I4X8fEYcQyOCWLUvFxHYd6KEsqpafjh+AI8uSCSitz8AxRU1/HT5Do4WlDM9PpzV+0/j7SX86xeT+SDlOJ+knuSl2ycwf3S/FvOqP70oOFvNT1/bQfqpUm5JHsi8pGhyS6tYsSuHPceKGRkTzITBViHLK6tix9EiSitt3DMnnpc2HmZA317cNmUwUX382ZdTwmd7T3KqpIqfTY/jjzcktbpvRGSXMabZW/61IHQh9jrDn1cd4F/bvgeBaUPDefqWccx9aiMTBvXlTz8YzcZD+Vw+LIKXNh7m7e3H+ON/jCI5LozEfn3w9hLuf28PX6Tl8tiCRBZNGXzO/ya3LNtGSaWNt34xmbveSmH/yVLmj45h9f5cbHZDL19vROCTe2cwPDq4zXyLymv4fN8pqmx28sqqOXCqlAmDQvnfVybg493+s9UXNmTxt9UZeIlw/WX9ePqWcRcUBmMMaSdK+WhPDgdOlXL8TCUnSyq5OjGaNenWH25iv2A+vXcm3l7CP9Zk8NzXWQwM68WaX11BLz/vhnWVVNhYtukwr2w+QqCfDwP69iKvrIqCszUALLtjIlclRvPMukziI4P4wbgB2Ox13PTCVk4UV7LmgVkNRaSpVftO8av3U5k2NJycogpOFFfy4m0TmTPy3Dv0K2vs5+QDVjH6xRs72X2smPjIIN5bMo3I4MZt1NrrWHvgNHERQYyMufDIpiktCN2kILTkqa8O8ey6TPx8vKiprUMEjIEls4by2+sSz5m34Gw19727h62HC0nq34ffLRjFtPhwiitqmPjEWv7zingevGYEpVU27ly+g7QTJSxMHsjimUMI8vdhwbPf0Nvfm0/vnUlIoHVJtLLGzu8/TSPA15sH540gJNCXsiobtyzbTvqpUsAaj2JwWCCZeWeZMiSMZ28dT3SfAOx1hvKaWvoENH95dV9OCTe+sIXp8eGEB/nxSepJ/rloPNdf1h9jDDa7Yf/JEv7yxUF2HD2Dn7cXY2JDCA30Y9GUgVw5MponVqbzxtZs3r97GhMHhwJWcX1hfRZzRkYxekBIs9vOyivjqa8OUW2rIzLYnyERQSTHhTJxcFiz8x86Xcb1z33DyJhg+gb6EeTnzY+TY5k1LJLswnJu+OcW+oUEUFljp7SqltfuTGZKk6O6tlTW2Hnn2++5YWx/ovoEtHu582lB6OYFoeBsNQuXbWPCoFBumzKI9QfzKCyv4Y83JOHbzP/ExhhWfneKv6w6wMmSKm6eEMvUoWE8tOI7Pr5nOuMHWX80NbV1nK2uJSzIr2HZndlnWPTKdsYPDOWtxZMpr67lrrdS2HO8GC8RQgN9uXpUDIdOl7H3eDEv3j6RKUPDCPT1xsfbi4925/Dox2n4+3px96x4Pk09QXZhOa//bDLT4sOptdfhJYKXl1BWZePHL22jqKKGNb+6gt4BPtzwz28oOFvN7VMG8/KmI5RVW0+JRvT24945Cdw0PrahUDVVUmFrdnpne2PLUf6+5hCDwgIbjir8fbwI8PXGx0tYed9MooMDqKq1N9uO4wpaELp5QbhYVTY7z32dyfPrD+Pv40VwgC87fjsXrzYapf699yT3vbeHoRFBHC+qBOCZW8YxKDyQv36Zwb6cYspr7Dx50xhumhB7wfKH88/y6/dT2ZtTwuDwQLxFyC2t4qYJA/h0z0l8vIWxA/uy8+gZymvsvHZnckO7xe5jRdz0wlYArkqMYtzAvvQN9OPG8QPo7e9ZbeQ2ex1fH8xjx9EzHDpdxn/NSejQEYGzaEHQgtCq17cc5fF/p3Pr5EH85aYx7VrmX9u/57l1mcwfHcMdUwcz7Lw2BZu9rtmjk6av78w+Q/LgMIorali4bBvHzlRw7eh+BPh6s/tYERMGhfLTaYMZO7DvOcuu2neK0EA/psW7/4+rK9KCoAWhTXuPFzM0MojgFs7lna2k0kaVzU70JZwbq/ZprSB41jGWcpvz/xd2tZBevvrchgfQOxWVUg20ICilGmhBUEo10IKglGrQI64yiEg+8H0LL0cAntTDqebTOs2nde3JZ7AxJrK5F3pEQWiNiKS0dAnGHTSf1mk+rbvUfPSUQSnVQAuCUqqBFgRHV+0eRPNpnebTukvKp8e3ISilGukRglKqQY8uCB0ZhdpJ2x8oIutF5ICI7BeR+x3Tw0TkKxHJdPwb6sKcvEVkj4isdMRDRORbRy7vi4hfW+voxFz6isgKETno2EfT3LxvHnB8Tmki8q6IBLhy/zQ3mnpL+0Mszzq+29+JyIT2bKPHFoSWRqF2cRq1wH8bYxKBqcC9jhweAdYZY4ZhjW7tymJ1P9Z4GfX+CvxfRy5FwGIX5vIM8KUxZiQw1pGXW/aNiAwA7gOSjTGjAW+sQYZcuX/e4LzR1Gl5f1wLDHP8LAFebNcWjDE98geYBqxuEi8Flro5p0+Bq4EMoJ9jWj8gw0Xbj3V8qa4EVmKNw1kA+DS3z5ycSx/gKI52ribT3bVv6gcpDsN6SnglcI2r9w8QB6S1tT+AZcCtzc3X2k+PPUKgg6NQO5uIxAHjgW+BaGPMKQDHv1EtL9mpngZ+A9Q54nCg2FijbYFr99FQIB943XEK86qIBOGmfWOMOQH8HTgGnAJKgF24b//Ua2l/XNT3uycXhA6NQu1MItIb+BD4lTGm1E05XA/kGWN2NZ3czKyu2kc+wATgRWPMeKAc1546ncNxbv4DYAjQHwjCOiw/n6dctruoz64nFwSPGIVaRHyxisE7xpiPHJNPi0g/x+v9gDwXpDIDuEFEsoH3sE4bngb6ikh9Rzqu3Ec5QI4x5ltHvAKrQLhj3wBcBRw1xuQbY2zAR8B03Ld/6rW0Py7q+92TC0LDKNSOluGfAJ+5MgGxBhd4DThgjHmqyUufAXc6fr8Tq23BqYwxS40xscaYOKx98bUx5jZgPfAjV+biyCcXOC4i9SPKzAXSccO+cTgGTBWRQMfnVp+PW/ZPEy3tj8+AnzquNkwFSupPLVrligYZT/0BrgMOAYeBR92w/ZlYh3HfAamOn+uwzt3XAZmOf8NcnNdsYKXj96HADiAL+H+AvwvzGAekOPbPJ0CoO/cN8DhwEEgD/gX4u3L/AO9itV/YsI4AFre0P7BOGZ53fLf3YV0daXMbeqeiUqpBTz5lUEqdRwuCUqqBFgSlVAMtCEqpBloQlFINtCAotxOR2fVPVyr30oKglGqgBUG1m4jcLiI7RCRVRJY5+k44KyL/EJHdIrJORCId844Tke2OZ/E/bvKcfoKIrBWRvY5l4h2r792k74N3HHcDIiJPiki6Yz1/d9Nb7zG0IKh2EZFE4BZghjFmHGAHbsN6yGe3MWYCsBH4g2ORt4CHjTGXYd0pVz/9HeB5Y8xYrGcB6m+nHQ/8CqtviqHADBEJA34IJDnW84Rz36XSgqDaay4wEdgpIqmOeCjWo9LvO+Z5G5gpIiFAX2PMRsf0N4FZIhIMDDDGfAxgjKkyxlQ45tlhjMkxxtRh3cIdB5QCVcCrInITUD+vchItCKq9BHjTGDPO8TPCGPPHZuZr7V745h7JrVfd5Hc7VqcjtcBkrKdBbwS+7GDOqoO0IKj2Wgf8SPeokvsAAADESURBVESioKEvv8FY36H6p/0WAd8YY0qAIhG53DH9DmCjsfp6yBGRGx3r8BeRwJY26OgnIsQYswrrdGKcM96YauTT9ixKgTEmXUR+B6wRES+sJ+7uxeq4JElEdmH1InSLY5E7gZccf/BHgJ87pt8BLBORPznW8eNWNhsMfCoiAVhHFw908ttS59GnHdUlEZGzxpje7s5DdQ49ZVBKNdAjBKVUAz1CUEo10IKglGqgBUEp1UALglKqgRYEpVQDLQhKqQb/Hz8vB7z7AcBiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 1.4336505\n"
     ]
    }
   ],
   "source": [
    "fit_and_plot_tf2(0, lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-12T05:44:06.603222Z",
     "start_time": "2020-04-12T05:43:59.719137Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAC1CAYAAAB24uKEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2dd3xUVd7/32dSSSEVaQESutI7CCoWlGJj18UVuzzC4+Ourq6I7K7rYlt3fxZkRUSkWFhsgCIiQui9BAKEmoSWAqRAElImZfL9/XEnTZKQQDLjzD3v12tec++555z7mTP3c+85555zrxIRNBqNe2NxtgCNRtP4aKNrNCZAG12jMQHa6BqNCdBG12hMgDa6RmMCPJ0t4GoIDw+XyMhIZ8vQaJxOTExMhog0q2m7Sxs9MjKS3bt3O1uGRuN0lFKnatvuklV3pdRdSqmPs7OznS1Fo3EJXNLoIvKDiEwMCgpythSNxiVwSaNrNJr64dJt9Mvxj2UHiQhpwv/c0N7ZUjSNTHFxMcnJyVitVmdLaVR8fX2JiIjAy8urXunc2ujbEjOJDPfTRjcBycnJBAYGEhkZiVLK2XIaBREhMzOT5ORkoqKi6pXWravu/j4e5BXanC1D4wCsVithYWFua3IApRRhYWFXVGtxSaPXtdfd38eT3MISB6nSOBt3NnkZV/obXdLode11D/DxJE8bXeMAsrKy+PDDD+udbvTo0WRlZTWCoqq4pNHrSp/iWLoW7HW2DI0JqMnoNlvtTccVK1YQHBzcWLLKcevOuFEZ8+hZrIA/OVuKxs156aWXSExMpHfv3nh5eREQEEDLli2JjY3l0KFD3HvvvSQlJWG1Wnn22WeZOHEiUDG6Mzc3l1GjRjFs2DC2bt1K69at+f7772nSpEmD6HNro6/s/Bpztp9hh4gp2m8ag2k/HORQak6D5nldq6a8cle3Gre/9dZbxMXFERsby/r16xkzZgxxcXHlvePz5s0jNDSUgoICBgwYwG9/+1vCwsKq5BEfH8+iRYuYM2cO48aNY/HixTz00EMNot+tq+4lQe1Ik2CsxaXOlqIxGQMHDqxyC2zGjBn06tWLwYMHk5SURHx8/CVpoqKi6N27NwD9+vXj5MmTDabHra/orYtP8pDHanILhtPE29/ZcjQOorYrr6Pw96843tavX090dDTbtm3Dz8+P4cOHV3uLzMfHp3zZw8ODgoKCBtPjklf0ut5ea5uzh9e95mPNOucgZRqzEhgYyMWLF6vdlp2dTUhICH5+fhw5coTt27c7WJ2LGr2ut9cyO9zLQOtMsj1CHKRMY1bCwsIYOnQo3bt3Z/LkyVW2jRw5kpKSEnr27MnLL7/M4MGDHa7PravuTQJCSCOE3CLdRtc0Pv/973+rDffx8eGnn36qdltZOzw8PJy4uLjy8BdeeKFBtbnkFb2uBFqKeNhjFersfmdL0Wicilsb3c9b8ZrXAgJSNjlbikbjVNza6P4BwfS3zmJ/m4edLUWjcSrubXRfTzIIIrdIv19OY27c2+jenoy2bKft6SXOlqLROBWXNHpd76NbLIr7vLbSK+VLBynTaH6duKTR6/NwyH94Pcf7HT5xgCqNmbnSaaoA06dPJz8/v4EVVcUljV4fPH0DuKjvo2samV+70d16wAxAN4/TjDy7BQr+BU0af96vxpxUnqY6YsQIrrnmGr7++msKCwsZO3Ys06ZNIy8vj3HjxpGcnIzNZuPll1/m3LlzpKamcvPNNxMeHs66desaRZ/bX9HbW85wR863kKvHu5uK+WNg70Jj2VZsrO/7ylgvyjfW4xYb69ZsY/3QMmM9L9NYP2ofzXbx8sfOW2+9RYcOHYiNjWXEiBHEx8ezc+dOYmNjiYmJYePGjaxcuZJWrVqxb98+4uLiGDlyJM888wytWrVi3bp1jWZyMIHRDwXdxN3BS6FZF2dL0ZiEVatWsWrVKvr06UPfvn05cuQI8fHx9OjRg+joaKZMmcKmTZtw5AtI3L7q7ufrQ15xw03307gIj/9YsezhVXXd26/qum9Q1XX/sKrrgc3rtWsRYerUqUyaNOmSbTExMaxYsYKpU6dy++238/e//71eeV8pbn9FD/SGJ/LnQ/xqZ0vRuDGVp6necccdzJs3j9zcXABSUlJIS0sjNTUVPz8/HnroIV544QX27NlzSdrGwu2v6P4+PtxVuhaSr4VOI5wtR+OmVJ6mOmrUKMaPH8+QIUMACAgI4IsvviAhIYHJkydjsVjw8vJi1qxZAEycOJFRo0bRsmXLRmunKxHXHR7av39/udxrk2esiee91UeIf2MMnh5uX4ExLYcPH+baa691tgyHUN1vVUrFiEj/mtK4/ZHv7+OJYNFvbNGYGpc0en3ejx7g48Gtlhg8l//BAco0ml8nLmn0+gyB9ffxpK1Kwytpq3H/VKMxIS5p9Prg7+PJfNso4n630bitonFbXLm/qa5c6W90e6MH+Bg3FnKt+h1s7oyvry+ZmZlubfay1yb7+vrWO63b314L8PHEhyI6bHoOSn4H193jbEmaRiAiIoLk5GTS09OdLaVR8fX1JSIiot7pTGH0QrwIvHAQcm9ythxNI+Hl5VXlzSiaqrh91d3fxxNQLBmyBAY+6Ww5Go1TcHujB/oalZYc3UbXmBi3N7qXh4VAH0/juXELxzlbjkbjFNze6AAh/t4UWa1QlGvMTdZoTIY5jO7nxY8+o+DxFcaURY3GZJjC6MF+3lzIL3K2DI3GaZjC6CF+XuTm5cH80bB7nrPlaDQOxxxG9/cmPR/w9gcPn8vG12jcDbcfMAMQ4ufNxcISiu7/Cm9PU5zbNJoquORRX59pqmBU3QGyCnQ7XWNOXNLo9ZmmCkbVHaDkwFJ4txvkn29MeRrNrw6XNHp9CfEzjH5BhUDkUCgpdLIijcaxmKKNHmyvuicF9qLbkJFOVqPROB5TXNFD7VX3C/n2UXFuPGdZo6kOUxi9rOp+Pq8IPv8NfP2IkxVpNI7FFFV3Xy8PfL0sZOUXQYebwbP+T+jQaFwZUxgdINTP26i6j/mjs6VoNA7HFFV3sI93z7PfRy8ugJwzzhWk0TgQ0xg9xN/LmNgiAh8Ohp//4mxJGo3DME3VPdjPmzNZOaAU3PxXaNrK2ZI0GodhGqOH+nlzvmyqak/9pBmNuahT1V0p9axSqqkymKuU2qOUur2xxTUkIX5eZBcUYyu130PPOg1xi50rSqNxEHVtoz8hIjnA7UAz4HHgrUZT1QgE+3kjAjkF9kEzu+bCkklQkOVcYRqNA6ir0ZX9ezQwX0T2VQpzCcpGx5VX3wf9L/xxNzQJdqIqjcYx1LWNHqOUWgVEAVOVUoFAaePJanjKxrtnlRm9aUsnqtFoHEtdr+gTgJeAASKSD3hhVN9dhjB/48kymbmV5qRfOAXf/wEyE52kSqNxDHU1+hDgqIhkKaUeAv4G1O2pD78SQgMqjXcvw+IBh76HtMNOUqXROIa6Gn0WkK+U6gW8CJwCPms0VY1AmL2NnlnZ6EERMDkRrr3TSarqScEFKMpztgr3RgRObASbA97sc+GUw94zUFejl4jxPtp7gPdF5H0gsPFkNTy+Xh74e3tUrboDeBonAOIWw8a3jWVbCeSmOVZgGXkZNf/588fAAgeelPIy4cLJK09fmAvF1svHEzGezluUXzXcVgxfPQyb3rlyDfVFSmHXJ7Dlvbqnycs0TFvGyc1wfP2l8UoKYc4tsPQpo2zmjzaajg6grka/qJSaCjwM/KiU8sBop7sUYQE+ZObV8HSZhLWQsMa4Ys67HVZObVwxpdX0ZV44BdN7wunt1afp/7gx+67yfHqRivUTm2Du7XD4h6r72fQO5FZ6nfCSSbB5evX7KMur1AZzb4Mf/lQRnry7Il52coWJRSAntWo+B76Ff0ZAwmpjPf887JwDJfYTbcwCWPOasXx6Gyx/Dg4vM9YLc+HiObB4Ql46WHMu1Zmbdvmr4bp/GifHX5Z1aSlseR82vVsRtusTSIg2mnNe/tAktGqa0zvg6E/GPm0lxglo24fGtqyTMKM3HFlhbFv+PGy2nyiKC+DbJ4z/1NMHIodB1I3gEwC3/BWG/J8x7yJ2kRG/pMj4/WCcQN7tBns+r/131oG6Gv1+oBDjfvpZoDXw/6567w4m1N+7ahu9MmPegUe+Nx4J3XEE9HvMCD9/HLZ+YPxhJYXG2fjYz9XnkZFQcfCn7oVPbqto/x/7GaL/YSxbc2D2DXDwu6rpLR7QZgCEtjfW93xuVCPBMNPAJ+HWvxvDeMv28V53SNpZkYeHN3QeVbF+JhbWvgHHfjIO7LwMCOsAB5cYZj7wbYXmHbONK46IoeWOf8JI+3CJ/V/DJ7dW6Fn2DMwdYSzvnAMzB0H6MWMfaUeg+2/h3lkQ3sWIE7cYVkyG8/aOz7MH4NQWY7nd9fDkWuhhH7G46W34YADkZ8ITK2HENENrdrKxvdhq1GyW/q9R41hwp2GKsv8rK8lYDmxh/FaLxTD3hZPGidxiAWu20T9TxukdsOJFI5+xs2DAhKr/TcJqWGx/G6+Hp5F3tn0/LXrCDX82TsIenvDgN3D/QvvvjIPU2Iom14hXofcDxnLv8dCyF2yZDj9NMU6Gy/8EC8YYx5q3n3FSCGrNVSMidfoAzYE77Z9r6pquMT/9+vWT+jBhwU4ZNX1jvdLI5vdFpoWKFGSJFOaKzL7JCBMRiV0ksve/xnJ2isjrLURWv2KsH99gxM3LNNajXxWZ0U+kqEAk56zIp3eLnNpuT5sqkrhexGar2K/NZsRfeL+x/b0eIglrjfD4aJFiq0j+BZFF40WSdxtpSkuNj4jI+RMiZ+OM5YwEkXOHRaaFiez8xEhrs4mk7hN5panI1g+MePu/Efl2gog159JyKCoQ2TmnQmN8tMjB743lC6dFoqeJXEwTeec6kYPfXZq+tLRCz+VIjxfZNqtq2pjPRF5tJlJSbITtni9yYpPI2YMib7Yxyl9E5OtHRT4YZJRNZdKOirwZIXJ4eUWetVFaKvLd08b/KGL89yl7Lo3TEBRbjf9CROToSpHN0+udBbBbavNvbRvLI8E4jA64TzE64U4A99UlbV0/gL89/znAg3VJU1+jT/4mVga+sbpeaUREJP1YxXJxYcXygrsMw5axa55Izpnq8yjME7GVVKxXPkg2/FvkPwMMw1SmqMA4wNKOiHx2r2HY4xsNc6bsrVlvXqbIv9qLLJlUNTwj4dK4ieur6rpazuwXObKi4fITMcpq5yeG2Yutl27POStivWgsZyYaJ9+SoqpxUvaK7F14aRnXxMmtIm93Ednz+dVpdxANZfR9la/iGMNg99Uh3TwgDYj7RfhI4CiQALxkD3sYuMu+/FVddNXX6G/9dFg6/uVHKW2oM3Fp6aVXjiuhKF/kwOLazVuGzSay8R2Rc4dqj3f24OXjaNyGyxm9riPjLCJSuRs6k7q17xcAH1DpVpy9I28mMAJIBnYppZYBEcABezRbHXXVizB/b4ptQo61hKAmDdCXqFTDDKH1agLdf1O3uBYL3PD85eM1v+7qNGnciroafaVS6mfA3jXI/cCKyyUSkY1KqchfBA8EEkTkOIBS6kuM23bJGGaPpZEeiFE+3j2vqGGMrtG4CHUylIhMBj4GegK9gI9FZMoV7rM1kFRpPdketgT4rVJqFvBDdQkBlFITlVK7lVK709PTa4pWLWEBxjDY8zXdYtNo3JQ6P3hCRBYDDTGBu7pZbyIiedRh/LyIfIxx0qF///71ekB72ei4jF8OmtFo3Jxaja6UughUZyaFYc6mV7DPZKBNpfUIILWGuA1KWHXj3TUaE1Cr0UWkMYa57gI6KaWigBTg98D4RtjPJZS10TNzddVdYy4a9SmwSqlFwDagi1IqWSk1QURKgD8APwOHga9F5GA9863Xa5PL8PH0INDHs+rEFo3GBDTqwyFF5IEawldQh177WvL9Afihf//+T9Y3bWiA96UTWzQaN8c0z3UvI6y28e4ajZtiOqOH+vvoqrvGdLik0a+0jQ4QHuCtO+M0psMljS4iP4jIxKCgoHqnLZuqagwP1mjMgUsa/Wpo3tSXklLRg2Y0psJ0Rm8b6gfA6fP62Wsa82A+o4cZRj+VmX+ZmBqN++CSRr+azriIkCYopY2uMRcuafSr6Yzz8fSgVVATTp/XRteYB5c0+tXSNtSPU5m6ja4xD6Y0erswP11115gKUxq9bZgfmXlF5BY64G0cGs2vAFMavV2oP4CuvmtMg0sa/Wp63cGougOc1tV3jUlwSaNfTa87VLqXrnveNSbBJY1+tTT19SLU31tX3TWmwZRGh7JbbPqKrjEHpjW6vsWmMROmNXpUuD+p2QVYixvlpTAaza8KUxtdBD0UVmMKXNLoV3t7DSAyzLiXfiJDd8hp3B+XNPrV3l4DiAzXRteYB5c0ekMQ1MSLMH9vTmqja0yAaY0OxlVdX9E1ZsDcRg/z56QeNKMxAaY2elS4H+dyCskv0rPYNO6NyY0eAMDJDH2LTePemNrokeHG5BbdTte4O+Y2uv1eum6na9wdlzR6QwyYAfD38aRlkC/7k7MaSJlG8+vEJY3eEANmyrirVyuiD6eRmlXQAMo0ml8nLmn0huThwe0QEb7YfsrZUjSaRsP0Rm8T6seI65qzaOdpPZNN47aY3ugAjw+N4kJ+Md/EJDtbikbTKGijA4OiQunXLoSZaxP0VV3jlmijA0op/nx7Z87mWHVbXeOWaKPbub5DOEM7hjFrfSKZuYXOlqPRNCja6JWYMrIrFwtLuPfDLXyzO4nffbSVP3+9z9myyjmTXUBWfhEAIsK2xExKS+Wy6ZLO57Nkj+5/aCx2HM9k0c7TzpZRK9rolegZEcxXEwdTUFTK5G/3sy8pm6V7k8mwX+ET0nLJLiiuNY/M3EJKbKWXhIsIi2OSax1uaysV5m4+Ue0ceRFh3OxtTFm8H4B1R9N4YM52Fu6ovamRV1jCY/N38vzX+0i+4Lgx/bPWJ/LuqqPYajkRrYw7S8yp8w7TVEaJrZRDqTmIXKotO7+Yp/+7p84v9ygosvHsl7H8dekBzmZba41rLbbxxIJdzFyXcNl8swuK+WBtfIP9Z9rov6BP2xCW/3EYsx7sy+KnrqdUYNXBcxxIzmbEexvo8+oqHvh4OxfyjCtrdkExWxIy2H48k6cX7qHf69H0nLaKCQt2VRmEM3vjcf78zT4enruj/Kr8S/618givLT/Eq8sPXbLtQEo2SecL2ByfQbGtlI3HMgB4LzqeHKtx8hERUrIKKCiyla//dekBEtONE8e6o+kAnMzIIzvfSHMhr4g3fjzErPWJTI8+xsjpG5mwYBfWYhsiwsHU7GrNaisVPtt2koS0iwDkF5Ww9sg5RISL1mLeiz7GjLUJTPxsd7VNob2nL/B/C2OY+FlMuf68X7wLr8RWyrbEzMsaqIzKxo1NyuLdVUcpKim9JM6Li/czesYmPtl04pI8Pt12kh/3n2FF3JnyfD7derLGfc7ZdJyzOVZKBb6NSapV2+Rv97P2SBofrU+koMhGbFIW/V+P5vCZnCpxd508z+j3N/H2qmM891VsnWptl8PzqnNwAkqpu4C7Onbs2Cj5twjyZVSPlogIUeH+/BR3hg3H0gj08eThIe2YtT6RjzYk8tyIztzzwWZO2s/+ft4eTLqxPQXFNpbuSWHsh1t4/d4exKdd5N8rj3J9hzB2nTzPn76K5bnbOhPs50U7+3j7hTtO8fHG47QObsK6o2mcyswjr9DGsn2pPD+iMz8fPAtAnv0A2ZyQQWSYHycz83nl+4NYlGLDsTQycosY26c1793fm43xGXwXm8pzt3Vmyd5k1h1J457erbjrg83c2KkZMx/syxfbTzHHfsArBT1bB7HmSBrPLNqLp4dixYGzjB/UljfH9igvH2uxjWe/3MvPB8/RrVVTfvjDMP654gifbz/FFxMGkZ5rpaiklIcGt2XRziQGvBFN/8hQ/vmbHnRoFoC12MYL3+wjxM+bzLwiPlyXSFATL95edZR5jw3gps7NWLYvldeWHyL9YiE3dArn8wmDKCiyMXtjIglpuVzIL6KopJRBUWH8+fbOJKbn8ei8nTwxLIrf9m3NpM93cy6nkENncpj5YF98PD0AeH9NPEv2pBAZ5scbKw6TV1RCfpGNbq2acke3Fiywm7psWPQHaxOIPnyOsABv7uzZqspxcia7gI82JDKqewuyC4r5ancS4/q34cXF+8nMLaJFkC/T7u5Gq+AmfLzxOD/sS2VU9xb8FHeWn+LOsHRvChm5hXy0IZH3f98HgIvWYp5YsItQf28m3dSe2RuOs2jXaR4c1O6qjmmXNLqI/AD80L9//ycbcz9KKUZ1b8HsjcexlQrP3NKR52/vwpksK59uO0l+kY2Tmfm8ObYHLYN86RERRHiADwDjB7XlsXm7ePKz3QAMbh/KvMcG8M3uJF7+/iDr7VfXSTe1p1mAD6//eJjhXZrx5tge3Pjvdby/Jp5tiZmcybYS6u/FzwfP0b11Uw6l5rA4JpmEtFz+MrorR85cZMneFAJ9PLn12ms4l1PIyrizvDG2hO/3ptDU15OnhnfgQn4RX+46zYItJ7loLWHVobNcyCtiaWwKg6JCmfvYAIpLSgnx92b+lhNM++EQXh6KIe3D+O+O03RpHkjzpj5sis9g3ZE0zuRYGdmtBSsPnuWd1UfLmxCzNybi42mhVZAvr97dnUeGRPLj/jN8sf0Uj8zdyZxH+vOftfEkpufx6RMD+T42hY83JlIqYFHw7upjdGvVlL8uOUDbMD/6tQ1h1aGzpF8sZPGeZKZHxxMZ5keovze2UuGDdQl4WBQ/HzxLSlYBry0/xDe7k8jMLWLCsCjmbj7BpM9j+PDBviyLTWV6dDz39Yvg9Xu788jcnUyPjseioFRgYFQo5/OKaBvqx76kbEpLhd32psXfvotjYGQo1zT1BWBLQoZxtRVhysiu7EvO4tkvYxk9YxN5hTYGtQ9lc3wGf1l6gH/c1Y13Vh9jZLcWzBzfl1veWc+7q4+RfKGAlkG+LN9/hhdHdqV1cBO+2pXERWsJX0wYRM+IIA4kZ/PWiiPc2rU5LYJ8r/hYdkmjO5JR3Vvy4fpEmnh58NjQKACeva0Ty/al8vn2U4zp0ZLxg9pekq5ri6b8+Mww9idn076ZP21C/LBYFA8Nbkf31kFcyC9i9aE0Zm84DsDIbi2Y/vve+Hp5MKpHS5bsScHH00KviCDeWXWMwpJSpt3djSWWlPKBPUM7hvPAwLbc2asl13cIx9fLg62JGYyfs4OVcWdZdegco3u0wNvTwvAuzViw9SQz1sSX1wTeWHGY4+l5PHlDewJ8PME4R/H40ChaBvnSNtSfzs0DeHT+Tl5ZdhAAf28Pru8Yzhu/6cFNnZox5j+bmbkukaa+ntw/oA1zNp3Aw6J4/PpILBZF5+aBdB4RyIjrmvP7j7czesYmPC2KF0d24abOzejcPIC1R9K4tWtzercN5uXv4piwYBf5xTbe/30fRISVB8+ybF8qX2w/xaCoUL6aNAQwqsPPfhnL+2viAfjkkf58vv0UG46lM/mOLjx9c0c6NAvgb98d4O4PtnA8PZfhXZrxz9/0wMvDwsInB3EqM4+IED9eWryf72JT6d0mmNE9WvDmiiNsP5FJVn4xTw3vwPwtJxjx3ka6t27K6fP5JJ0voEMzfxY8PpDIcH9aBPkS7OdFfpGNBY8PYFD7MOZuPsFryw/x4Lkd+HhYmHZPNywWxbgBbfj3yqME+3nx2RMDGfn+JuZvPsFLo7oyf8tJBkaG0qtNMABvju3Bc1/HkltYDGijNxrdWzelX7sQbugUTqi/NwDtwvx5cFBbluxN4W93Xltj2rAAH27uek2VMKUUfdqGAHBL1+bc0Cmc4+m5PDW8Ix4WBcD/DIti7eFzTLunO/3ahXDHexsBuL1bc9IvFrIvKYtQf2+ubdEUi0VxS9fm5fkPigojPMCHN1ccJrewpLy6Obh9GL5eFqzFpbxydzfe/vko38Yk4+1hYXT3lpdoH1kpbOb4vizdm0LPiCB6RgTj5VHRtTN1VFcembeT50d0ZmyfCBbuOE1+kY0xPavm2b11EHMf7c8XO07zh5s70qVFIAAtg5qw8y+34e1pobDExqx1CexLzubhwe3oeI3xYJCuLQKZHn2Mi9YSpo6qKG+lFP++ryf5RTYGRIZw23XNGdYpnK2JGdzU2Sj38YPaEh7gzR8X7aVXm2A+fLBvuX4vDwsdrzF0vDOuNz0jghnaMZwL9j6UeZuNJs0DA9pyU+dmLN2TwqEzOXRt0ZQJQ6MYN6ANft6GhXy9PPj8iUH4elno1NzI87HrI/lubwoHUrKZdnc3mttrA/f1jeA/axKYdGMHOjUP5M6eLfls2ykOn80hJauAV+66rvw3Rob7s+Sp61FKXfIf1QsRcdlPv379xFnYbKWSlV/UaPkXFtvKl+dsTJTJ38SKiMi2xAxpN2W5PL0wpsa0L393QNpNWS59Xl0lxSUV+Ty9MEbGzNgopaWlMn/zcWk3Zbk8+emuq9Z6JqugfPndVUfl7v9sktLS0ivK69vdSTL0rTWScdFaHvbhugRpN2W5DHxjtRRV+j314Vx2gViLS+oU96K1WCJfWl6+zyv9LSIiJ9Jz5aP1CVJiq5pHVl5Reb5pOVZ58Zt9cu3LP8lt76wXm63++wN2Sy1ecbpZr+bjTKM7i8Jim9w/e6usO3Kuxjg7jmdKuynL5S9L9l+StqDIONgv5BXK7e9ukC3x6Y2qtyFIOp8nHab+KDOijzlsn7e9s/6yJ9SGJtdaLBetxVeU9nJG11V3F8Pb08KXE4fUGqd/uxCmjOzKXb2qVp+9PSuq3MF+3vz83I2NorGhiQjxI/r5m4gIaeKwffaMCCY+LZdBUaEO26e/T+PZUd9Hd0MsFsVTwzsQEeLnbCkNRmS4P54ejjtc+7YzOsMGtQ9z2D4bE31F12iq4Xf92tChWQCd7R1rro6+oms01eDtaWGwm1zNQRtdozEF2ugajQnQRtdoTIA2ukZjApRUMyfXVVBKpQM1TcgOBzIcKOdyaD21o/XUTF20tBORZjVtdGmj14ZSareI9He2jopw1RgAAAVfSURBVDK0ntrRemqmIbToqrtGYwK00TUaE+DORv/Y2QJ+gdZTO1pPzVy1Frdto2s0mgrc+Yqu0WjsuKXRlVIjlVJHlVIJSqmXHLzvNkqpdUqpw0qpg0qpZ+3hoUqp1UqpePt3iIN1eSil9iqlltvXo5RSO+x6vlJKeTtQS7BS6lul1BF7OQ1xZvkopZ6z/1dxSqlFSilfR5aPUmqeUipNKRVXKaza8lAGM+zH9n6lVN+67MPtjK6U8gBmAqOA64AHlFLX1Z6qQSkB/iwi1wKDgaft+38JWCMinYA19nVH8ixwuNL6v4D37HouABMcqOV9YKWIdAV62XU5pXyUUq2BZ4D+ItId8AB+j2PLZwEw8hdhNZXHKKCT/TMRmFWnPdT2VApX/ABDgJ8rrU8FpjpRz/fACOAo0NIe1hI46kANEfaD5RZgOaAwBmB4VldmjaylKXACe/9QpXCnlA/QGkgCQjGmbS8H7nB0+QCRQNzlygOYDTxQXbzaPm53Rafijysj2R7mcJRSkUAfYAfQXETOANi/r6k5ZYMzHXgRKHubQRiQJSJlb0xwZBm1B9KB+famxCdKKX+cVD4ikgK8DZwGzgDZQAzOK58yaiqPKzq+3dHo1T0u0+G3FpRSAcBi4E8iknO5+I2o404gTURiKgdXE9VRZeQJ9AVmiUgfIA/HN2PKsbd97wGigFaAP0b1+Jf8Wm5PXdF/545GTwbaVFqPAFIdKUAp5YVh8oUissQefE4p1dK+vSWQ5iA5Q4G7lVIngS8xqu/TgWClVNkThhxZRslAsojssK9/i2F8Z5XPbcAJEUkXkWJgCXA9ziufMmoqjys6vt3R6LuATvZeU2+MjpVljtq5Mh7APRc4LCLvVtq0DHjUvvwoRtu90RGRqSISISKRGGWxVkQeBNYB9zlBz1kgSSnVxR50K3AIJ5UPRpV9sFLKz/7flelxSvlUoqbyWAY8Yu99Hwxkl1Xxa8URHR6O/gCjgWNAIvBXB+97GEZVaj8Qa/+MxmgXrwHi7d+hTiiX4cBy+3J7YCeQAHwD+DhQR29gt72MvgNCnFk+wDTgCBAHfI7xzhqHlQ+wCKN/oBjjij2hpvLAqLrPtB/bBzDuFlx2H3pknEZjAtyx6q7RaH6BNrpGYwK00TUaE6CNrtGYAG10jcYEaKNrGgWl1PCymXIa56ONrtGYAG10k6OUekgptVMpFauUmm2ft56rlHpHKbVHKbVGKdXMHre3Umq7fR700kpzpDsqpaKVUvvsaTrYsw+oNO98oX3kGUqpt5RSh+z5vO2kn24qtNFNjFLqWuB+YKiI9AZswIMYEzv2iEhfYAPwij3JZ8AUEemJMSqrLHwhMFNEemGMEy8bktkH+BPGcwHaA0OVUqHAWKCbPZ/XG/dXakAb3ezcCvQDdimlYu3r7TGms35lj/MFMEwpFQQEi8gGe/inwI1KqUCgtYgsBRARq4jk2+PsFJFkESnFGAocCeQAVuATpdRvgLK4mkZEG93cKOBTEelt/3QRkX9UE6+2cdLVTZsso7DSsg3jQQ4lwECM2X33AivrqVlzBWijm5s1wH1KqWug/Dll7TCOi7KZW+OBzSKSDVxQSt1gD38Y2CDGXPtkpdS99jx8lFJ+Ne3QPk8/SERWYFTrezfGD9NUxfPyUTTuiogcUkr9DVillLJgzJ56GuNhEN2UUjEYT1y5357kUeAju5GPA4/bwx8GZiulXrXn8btadhsIfK+U8sWoDTzXwD9LUw169prmEpRSuSIS4GwdmoZDV901GhOgr+gajQnQV3SNxgRoo2s0JkAbXaMxAdroGo0J0EbXaEyANrpGYwL+P7mdsrw7zV28AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 0.26387808\n"
     ]
    }
   ],
   "source": [
    "fit_and_plot_tf2(3, lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "3.12_weight-decay.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
